{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import json\n",
    "import csv\n",
    "import os\n",
    "import random\n",
    "from tempfile import gettempdir\n",
    "\n",
    "import numpy as np\n",
    "np.set_printoptions(threshold = np.nan)\n",
    "import pandas as pd\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "import shutil\n",
    "\n",
    "\n",
    "# tf.enable_eager_execution()\n",
    "tf.executing_eagerly()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 6680\n",
      "drwxr-xr-x@ 8 ryangillard  eng     256 Jan 18 11:57 \u001b[34m.\u001b[m\u001b[m/\n",
      "drwxr-xr-x@ 7 ryangillard  eng     224 Jan 18 11:58 \u001b[34m..\u001b[m\u001b[m/\n",
      "-rwxr-xr-x@ 1 ryangillard  eng  159159 Mar  1  2018 \u001b[31mARC-Easy-Dev.csv\u001b[m\u001b[m*\n",
      "-rwxr-xr-x@ 1 ryangillard  eng  214423 Mar  1  2018 \u001b[31mARC-Easy-Dev.jsonl\u001b[m\u001b[m*\n",
      "-rwxr-xr-x@ 1 ryangillard  eng  666020 Mar  1  2018 \u001b[31mARC-Easy-Test.csv\u001b[m\u001b[m*\n",
      "-rwxr-xr-x@ 1 ryangillard  eng  895104 Mar  1  2018 \u001b[31mARC-Easy-Test.jsonl\u001b[m\u001b[m*\n",
      "-rwxr-xr-x@ 1 ryangillard  eng  629638 Mar  1  2018 \u001b[31mARC-Easy-Train.csv\u001b[m\u001b[m*\n",
      "-rwxr-xr-x@ 1 ryangillard  eng  844113 Mar  1  2018 \u001b[31mARC-Easy-Train.jsonl\u001b[m\u001b[m*\n"
     ]
    }
   ],
   "source": [
    "ls -la ../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_loc = '../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy/ARC-Easy-Dev.jsonl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(file_loc) as f:\n",
    "    content = f.readlines()\n",
    "print(len(content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "content[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string = json.loads(content[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weird entries\n",
    "Some have numbered answers, fill in the blank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string = json.loads(content[8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write jsonl data to csv as labeled dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "regex = r\"(?<!\\d)[.,;:?](?!\\d)\"\n",
    "\n",
    "letter_answers = ['A', 'B', 'C', 'D', 'E']\n",
    "number_answers = ['1', '2', '3', '4', '5']\n",
    "\n",
    "def extract_question_answer(str_line):\n",
    "  \"\"\"Reads in line as string, returns tuple of question, answer\"\"\"\n",
    "  json_line = json.loads(str_line)\n",
    "  \n",
    "  question = json_line['question']['stem']\n",
    "  \n",
    "  choices = [choice['text'] for choice in json_line['question']['choices']]\n",
    "  \n",
    "  answer = json_line['answerKey']\n",
    "  if answer in letter_answers:\n",
    "    answer_idx = letter_answers.index(answer)\n",
    "  else: \n",
    "    answer_idx = number_answers.index(answer)\n",
    "  answer_text = json_line['question']['choices'][answer_idx]['text']\n",
    "  \n",
    "  question_formatted = \"<s> \" + re.sub(\n",
    "    regex, \"\", question, 0).replace(\"\\\"\",\"\").replace(\"\\'\",\"\").replace(\",\",\"\").replace(\";\",\"\") + \" </s>\"\n",
    "#   print(\"question_formatted = \\n{}\".format(question_formatted))\n",
    "  choices_formatted = \";\".join(\n",
    "    [\"<s> \" + re.sub(\n",
    "      regex, \"\", choice, 0).replace(\"\\\"\",\"\").replace(\"\\'\",\"\").replace(\",\",\"\").replace(\";\",\"\") + \" </s>\" \n",
    "     for choice in choices])\n",
    "#   print(\"choices_formatted = \\n{}\".format(choices_formatted))\n",
    "  \n",
    "  return question_formatted, choices_formatted, answer_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import codecs\n",
    "\n",
    "filename = \"train_data.csv\"\n",
    "file_loc = \"../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy/ARC-Easy-Train.jsonl\"\n",
    "\n",
    "if os.path.exists(filename):\n",
    "  os.remove(filename)\n",
    "  \n",
    "with io.open(file_loc, encoding='utf-8') as f:\n",
    "  with open(filename, \"w\") as out:\n",
    "    for line in f:\n",
    "      question, choices, answer_idx = extract_question_answer(line)\n",
    "      with codecs.open(filename, \"a\", \"utf-8\") as temp:\n",
    "        temp.write(question + \",\" + choices + \",\" + str(answer_idx) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import codecs\n",
    "\n",
    "filename = \"eval_data.csv\"\n",
    "file_loc = \"../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy/ARC-Easy-Dev.jsonl\"\n",
    "\n",
    "if os.path.exists(filename):\n",
    "  os.remove(filename)\n",
    "  \n",
    "with io.open(file_loc, encoding='utf-8') as f:\n",
    "  with open(filename, \"w\") as out:\n",
    "    for line in f:\n",
    "      question, choices, answer_idx = extract_question_answer(line)\n",
    "      with codecs.open(filename, \"a\", \"utf-8\") as temp:\n",
    "        temp.write(question + \",\" + choices + \",\" + str(answer_idx) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s> Which factor will most likely cause a person to develop a fever </s>,<s> a leg muscle relaxing after exercise </s>;<s> a bacterial population in the bloodstream </s>;<s> several viral particles on the skin </s>;<s> carbohydrates being digested in the stomach </s>,1\n",
      "<s> Lichens are symbiotic organisms made of green algae and fungi What do the green algae supply to the fungi in this symbiotic relationship </s>,<s> carbon dioxide </s>;<s> food </s>;<s> protection </s>;<s> water </s>,1\n",
      "<s> When a switch is used in an electrical circuit the switch can </s>,<s> cause the charge to build </s>;<s> increase and decrease the voltage </s>;<s> cause the current to change direction </s>;<s> stop and start the flow of current </s>,3\n",
      "<s> Which of the following is an example of an assistive device </s>,<s> contact lens </s>;<s> motorcycle </s>;<s> raincoat </s>;<s> coffee pot </s>,0\n",
      "<s> Rocks are classified as igneous metamorphic or sedimentary according to </s>,<s> their color </s>;<s> their shape </s>;<s> how they formed </s>;<s> the minerals they contain </s>,2\n"
     ]
    }
   ],
   "source": [
    "!head -5 train_data.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s> Which technology was developed most recently </s>,<s> cellular telephone </s>;<s> television </s>;<s> refrigerator </s>;<s> airplane </s>,0\n",
      "<s> A student hypothesizes that algae are producers Which question will best help the student determine if this is correct </s>,<s> Do algae consume other organisms </s>;<s> Which organisms consume algae </s>;<s> Do algae use sunlight to make food </s>;<s> Could an ecosystem survive without algae </s>,2\n",
      "<s> Soccer players use their muscle systems to kick a ball into a goal What organ system coordinates the muscles </s>,<s> The nervous system </s>;<s> The endocrine system </s>;<s> The respiratory system </s>;<s> The circulatory system </s>,0\n",
      "<s> Planets in the solar system are in constant motion What factor has the greatest effect on the orbits of the planets </s>,<s> the size of the planets </s>;<s> gravitational pull of the Sun </s>;<s> the composition of the planets </s>;<s> electromagnetic radiation from the Sun </s>,1\n",
      "<s> How is a pond different from a lake </s>,<s> Ponds have moving water </s>;<s> Ponds are smaller and shallower </s>;<s> Ponds are not surrounded by land </s>;<s> Ponds have a different amount of salt </s>,1\n"
     ]
    }
   ],
   "source": [
    "!head -5 eval_data.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learn embeddings from ARC corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Large international companies are involved in bauxite, iron ore, diamond, and gold mining operations.\n",
      "Paleoceanography, 8(2): 193-208.\n",
      "Of course, for many in the media, “hydrogen sulphide delivery helps prevent disease damage in cells in certain disease models” will always be trumped by “farts cure cancer” when it comes to headlines.\n",
      "The same problems apply with wolf-domestic dog hybrids.\n",
      "taking stock of delightful days\n",
      "The an- tlu-opologist and the ethnologist find in trop- ical America some of the most complicated and interesting problems of research.\n",
      "ORDER ODONATA (Damselflies and Dragonflies) Diagnosis: large, to over three inches long; four wings, transparent and membranous, held vertically (damselflies) or laterally (dragonflies) at rest; chew- ing mouth parts, tooth-like; nymphs aquatic, feeding on mosquito larvae to small fish; adults terrestrial, feeding on other insects (Figure 14.27).\n",
      "until they institute such safeguards and assurances of chaste maidenhood as characterize Hebrew social life?\n",
      "Sex brought the variations that could allow organisms to survive change” (2000, p. 163).\n",
      "Homo erectus had a long tenure; the earliest Homo erectus fossils are dated to roughly 1.8 million years ago, while the youngest fossils assigned to this species date to roughly 300 thousand years ago (ka).\n"
     ]
    }
   ],
   "source": [
    "!head -10 ARC_Corpus_small.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "266778143\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "regex = r\"(?<!\\d)[.,;:?`\\\"\\'\\(\\)\\[\\]\\{\\}\\\\/“”](?!\\d)\"\n",
    "\n",
    "# Step 1: Download the data.\n",
    "filename = \"ARC_Corpus.txt\"\n",
    "\n",
    "# Read the data into a list of strings.\n",
    "def read_data(filename):\n",
    "  with open(filename, \"r\") as ins:\n",
    "    sentence_list = []\n",
    "    for line in ins:\n",
    "      line = line.replace('\\n', '')\n",
    "      line = \"<s> \" + re.sub(\n",
    "        regex, \"\", line, 0).replace(\"(\",\" \").replace(\")\",\" \").replace(\"- \",\"\").replace(\"“\",\"\").replace(\"”\",\"\").replace(\"\\\"\",\"\").replace(\"\\'\",\"\").replace(\"  \",\" \") + \" </s>\"\n",
    "      line = line.split(' ')\n",
    "      line[:] = [word for word in line if (word != \" \" and word != \"\\n\" and word != \"\")]\n",
    "      sentence_list.append(line)\n",
    "      \n",
    "  vocabulary_list = [word for sentence in sentence_list for word in sentence]\n",
    "    \n",
    "  return vocabulary_list\n",
    "\n",
    "vocabulary = read_data(filename)\n",
    "print(len(vocabulary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common words (+UNK) [['UNK', 17625492], ('<s>', 14621860), ('</s>', 14621856), ('the', 11738977), ('of', 7788564)]\n",
      "Sample data [1, 3022, 1747, 1582, 12, 807, 8, 48802, 1485, 8321] ['<s>', 'Large', 'international', 'companies', 'are', 'involved', 'in', 'bauxite', 'iron', 'ore']\n"
     ]
    }
   ],
   "source": [
    "# Step 2: Build the dictionary and replace rare words with UNK token.\n",
    "vocabulary_size = 50000\n",
    "\n",
    "def build_dataset(words, n_words):\n",
    "  \"\"\"Process raw inputs into a dataset.\"\"\"\n",
    "  count = [[\"UNK\", -1]]\n",
    "\n",
    "  count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "  \n",
    "  unique_words = [word_tuple[0] for word_tuple in count]\n",
    "  \n",
    "  def fix_start_stop_maybe(count, unique_words, token_string, token_index):\n",
    "    index = unique_words.index(token_string) if token_string in unique_words else -1\n",
    "    \n",
    "    if index != token_index:\n",
    "      count.insert(token_index, count.pop(index))\n",
    "      unique_words.insert(token_index, unique_words.pop(index))\n",
    "    return\n",
    "    \n",
    "  fix_start_stop_maybe(count, unique_words, \"<s>\", 1)\n",
    "  fix_start_stop_maybe(count, unique_words, \"</s>\", 2)\n",
    "  \n",
    "  count[:] = count[0:vocabulary_size]\n",
    "  \n",
    "  dictionary = dict()\n",
    "  for word, _ in count:\n",
    "    dictionary[word] = len(dictionary)\n",
    "    \n",
    "  data = list()\n",
    "  unk_count = 0\n",
    "  for word in words:\n",
    "    index = dictionary.get(word, 0)\n",
    "    if index == 0:  # dictionary[\"UNK\"]\n",
    "      unk_count += 1\n",
    "    data.append(index)\n",
    "  count[0][1] = unk_count\n",
    "  \n",
    "  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "  return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "# Filling 4 global variables:\n",
    "# data - list of codes (integers from 0 to vocabulary_size-1).\n",
    "#   This is the original text but words are replaced by their codes\n",
    "# count - map of words(strings) to count of occurrences\n",
    "# dictionary - map of words(strings) to their codes(integers)\n",
    "# reverse_dictionary - maps codes(integers) to words(strings)\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(vocabulary, vocabulary_size)\n",
    "# del vocabulary  # Hint to reduce memory.\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])\n",
    "\n",
    "data_index = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3022 Large -> 1 <s>\n",
      "3022 Large -> 1747 international\n",
      "1747 international -> 3022 Large\n",
      "1747 international -> 1582 companies\n",
      "1582 companies -> 12 are\n",
      "1582 companies -> 1747 international\n",
      "12 are -> 807 involved\n",
      "12 are -> 1582 companies\n"
     ]
    }
   ],
   "source": [
    "# Step 3: Function to generate a training batch for the skip-gram model.\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "  global data_index\n",
    "  assert batch_size % num_skips == 0\n",
    "  assert num_skips <= 2 * skip_window\n",
    "  batch = np.ndarray(\n",
    "    shape = (batch_size), \n",
    "    dtype = np.int32)\n",
    "  labels = np.ndarray(\n",
    "    shape = (batch_size, 1), \n",
    "    dtype = np.int32)\n",
    "  span = 2 * skip_window + 1  # [ skip_window target skip_window ]\n",
    "  buffer = collections.deque(maxlen=span)\n",
    "  if data_index + span > len(data):\n",
    "    data_index = 0\n",
    "  buffer.extend(data[data_index:data_index + span])\n",
    "  data_index += span\n",
    "  for i in range(batch_size // num_skips):\n",
    "    context_words = [w for w in range(span) if w != skip_window]\n",
    "    words_to_use = random.sample(context_words, num_skips)\n",
    "    for j, context_word in enumerate(words_to_use):\n",
    "      batch[i * num_skips + j] = buffer[skip_window]\n",
    "      labels[i * num_skips + j, 0] = buffer[context_word]\n",
    "    if data_index == len(data):\n",
    "      buffer[:] = data[:span]\n",
    "      data_index = span\n",
    "    else:\n",
    "      buffer.append(data[data_index])\n",
    "      data_index += 1\n",
    "  # Backtrack a little bit to avoid skipping words in the end of a batch\n",
    "  data_index = (data_index + len(data) - span) % len(data)\n",
    "  return batch, labels\n",
    "\n",
    "batch, labels = generate_batch(batch_size = 8, num_skips = 2, skip_window = 1)\n",
    "# This is first doing target word with left word, then target word with right word\n",
    "for i in range(8):\n",
    "  print(batch[i], reverse_dictionary[batch[i]],\n",
    "    '->', labels[i, 0], reverse_dictionary[labels[i, 0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 4: Build and train a skip-gram model.\n",
    "batch_size = 128\n",
    "embedding_size = 128  # Dimension of the embedding vector.\n",
    "skip_window = 1     # How many words to consider left and right.\n",
    "num_skips = 2     # How many times to reuse an input to generate a label.\n",
    "num_sampled = 64    # Number of negative examples to sample.\n",
    "\n",
    "# We pick a random validation set to sample nearest neighbors. Here we limit the\n",
    "# validation samples to the words that have a low numeric ID, which by\n",
    "# construction are also the most frequent. These 3 variables are used only for\n",
    "# displaying model accuracy, they don't affect calculation.\n",
    "valid_size = 16   # Random set of words to evaluate similarity on.\n",
    "valid_window = 100  # Only pick dev samples in the head of the distribution.\n",
    "# Generate a uniform random sample from np.arange(valid_window) of size valid_size without replacement\n",
    "valid_examples = np.random.choice(\n",
    "  a = valid_window, \n",
    "  size = valid_size, \n",
    "  replace = False)\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "  # Input data.\n",
    "  train_inputs = tf.placeholder(dtype = tf.int32, shape = [batch_size]) # batch_size vector\n",
    "  train_labels = tf.placeholder(dtype = tf.int32, shape = [batch_size, 1]) # batch_size x 1 matrix\n",
    "  valid_dataset = tf.constant(value = valid_examples, dtype = tf.int32) # valid_size vector\n",
    "  #print(\"valid_dataset = \", valid_dataset)\n",
    "\n",
    "  # Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "  with tf.device(\"/cpu:0\"):\n",
    "    # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "      initial_value = tf.random_uniform(\n",
    "        shape = [vocabulary_size, embedding_size], \n",
    "        minval = -1.0, \n",
    "        maxval = 1.0)) # embedding weights are vocabulary_size x embedding_size matrix\n",
    "    #print(\"embeddings = \", embeddings)\n",
    "    embed = tf.nn.embedding_lookup(\n",
    "      params = embeddings, \n",
    "      ids = train_inputs) # embedding_size x embedding_size matrix\n",
    "    #print(\"embed = \", embed)\n",
    "\n",
    "    # Construct the variables for the NCE loss\n",
    "    nce_weights = tf.Variable(\n",
    "      initial_value = tf.truncated_normal(\n",
    "        shape = [vocabulary_size, embedding_size], \n",
    "        stddev = 1.0 / math.sqrt(embedding_size))) # nce weights are vocabulary_size x embedding_size matrix\n",
    "    #print(\"nce_weights = \", nce_weights)\n",
    "    nce_biases = tf.Variable(\n",
    "      initial_value = tf.zeros(\n",
    "        shape = [vocabulary_size])) # nce biases are a vocabulary_size vector\n",
    "    #print(\"nce_biases = \", nce_biases)\n",
    "\n",
    "  # Compute the average NCE loss for the batch.\n",
    "  # tf.nce_loss automatically draws a new sample of the negative labels each\n",
    "  # time we evaluate the loss.\n",
    "  # Explanation of the meaning of NCE loss:\n",
    "  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.nce_loss(weights=nce_weights,\n",
    "           biases=nce_biases,\n",
    "           labels=train_labels,\n",
    "           inputs=embed,\n",
    "           num_sampled=num_sampled,\n",
    "           num_classes=vocabulary_size)) # scalar\n",
    "    #print(\"loss = \", loss)\n",
    "\n",
    "  # Construct the SGD optimizer using a learning rate of 1.0.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "\n",
    "  # Compute the cosine similarity between minibatch examples and all embeddings.\n",
    "  norm = tf.sqrt(\n",
    "    x = tf.reduce_sum(\n",
    "      input_tensor = tf.square(\n",
    "        x = embeddings), \n",
    "      axis = 1, \n",
    "      keepdims = True)) # vocabulary_size x 1 matrix, summed along the embedding_size axis\n",
    "  #print(\"norm = \", norm)\n",
    "  normalized_embeddings = embeddings / norm # vocabulary_size x embedding_size matrix\n",
    "  #print(\"normalized_embeddings = \", normalized_embeddings)\n",
    "  valid_embeddings = tf.nn.embedding_lookup(\n",
    "    params = normalized_embeddings, \n",
    "    ids = valid_dataset) # valid_size x embedding_size matrix\n",
    "  #print(\"valid_embeddings = \", valid_embeddings)\n",
    "  # cosine similarity\n",
    "  similarity = tf.matmul(\n",
    "    a = valid_embeddings, \n",
    "    b = normalized_embeddings, \n",
    "    transpose_b = True) #  valid_size x vocabulary_size = (valid_size x embedding_size matrix) x (vocabulary_size x embedding_size matrix)^T\n",
    "  #print(\"similarity = \", similarity)\n",
    "\n",
    "  # Add variable initializer.\n",
    "  init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step  0 :  237.41680908203125\n",
      "Nearest to your: ropes, Copulation, Cathedral, auxiliary, manmade, Mitochondria, pigment, chaparral,\n",
      "Nearest to would: titanotheres, Have, parenteral, seawater, subduction, entertainers, IC, students’,\n",
      "Nearest to at: coalition, Beginners, 41:, decipher, Prasad, thirst, Robbins, antecedent,\n",
      "Nearest to between: permanence, Company, Scratch, tally, Magnificent, rejoice, monotypic, Werner,\n",
      "Nearest to like: Mare, Peter, artefacts, 1959., secondhand, MATERIALS, carnivores, funeral,\n",
      "Nearest to out: Café, HF, please, Peppers, backstory, flowered, ancient, certification,\n",
      "Nearest to most: barometer, brokers, helper, Carrier, apartments, 42., Zen, bank,\n",
      "Nearest to first: [134], extras, SUCH, coagulation, safely, waiting, -LRBdied, patrol,\n",
      "Nearest to into: Survival, recount, Hyde, Underwood, adjective, ci, acquire, Bunker,\n",
      "Nearest to when: Assad, spool, usefulness, dicots, tokens, depositing, trembling, Kittens,\n",
      "Nearest to these: Adirondack, esoteric, anniversary, lettuce, GF, tangle, paying, mais,\n",
      "Nearest to been: existent, Odysseus, Holdings, potable, endurance, Ficus, directs, Roach,\n",
      "Nearest to two: expedient, GC, Producers, declared, Meaning, hemisphere, shifting, Indoor,\n",
      "Nearest to found: 1840s, itchy, ulceration, Benny, trimester, signal, Psyche, ventrally,\n",
      "Nearest to to: declaration, guidance, Lindley, built-in, fungus, 106,, Lantern, monotonous,\n",
      "Nearest to years: Captivity, welfare, Forudesigns, participating, Tooth, dataset, recourse, refuted,\n",
      "Average loss at step  2000 :  117.88296962022781\n",
      "Average loss at step  4000 :  55.3640566072464\n",
      "Average loss at step  6000 :  34.27510245609283\n",
      "Average loss at step  8000 :  24.16776530957222\n",
      "Average loss at step  10000 :  18.139350516080857\n",
      "Nearest to your: grama, ropes, Asia, irradiance, mix, auxiliary, Cathedral, his,\n",
      "Nearest to would: Have, modification, medical, also, ever,   , shoots, was,\n",
      "Nearest to at: for, from, -RRB#, to, in, hectares, defeat, selection,\n",
      "Nearest to between: muscles, Company, Werner, excessive, giving, similar, sure, pattern,\n",
      "Nearest to like: carnivores, kid, king, Peter, long, funeral, Video, hectares,\n",
      "Nearest to out: HF, please, assessment, kid, show, ancient, lines, Navy,\n",
      "Nearest to most: identify, bank, United, Zen, apartments, Sigmund, budget, Video,\n",
      "Nearest to first: waiting, [134], safely,   , cause, -RRB#, acute, ninja,\n",
      "Nearest to into: Survival, ecological, acquire, sorts, first, and, Doctor, microwave,\n",
      "Nearest to when: for, cause, visit, receive, Sedum, by, if, nbsp,\n",
      "Nearest to these: the, Snape, a, relief, anniversary, shoot, paying, sticking,\n",
      "Nearest to been: Sedum, Later, Odysseus, lens, endurance, Guinea, may, Biology,\n",
      "Nearest to two: 0.00, spiders, shifting, GC, measures, declared, Meaning, hemisphere,\n",
      "Nearest to found: itchy, ulceration, Sedum, UNK, static, driven, trimester, 55,\n",
      "Nearest to to: and, in, for,   , -RRB#, with, wishes, can,\n",
      "Nearest to years: learn, welfare, y, touch, recourse, exchanges, Snape, participating,\n",
      "Average loss at step  12000 :  14.24176759839058\n",
      "Average loss at step  14000 :  11.817096354484558\n",
      "Average loss at step  16000 :  10.153853011846543\n",
      "Average loss at step  18000 :  8.77055635714531\n",
      "Average loss at step  20000 :  7.8516016952991485\n",
      "Nearest to your: the, his, a, or, irradiance, grama, notification, ropes,\n",
      "Nearest to would: Have, was, also, modification, can, medical, may, must,\n",
      "Nearest to at: for, in, and, from, to, -RRB#, defeat, hectares,\n",
      "Nearest to between: muscles, Company, permanence, Werner, teens, sure, pattern, hungry,\n",
      "Nearest to like: carnivores, funeral, Video, 1850s, hectares, after, artefacts, long,\n",
      "Nearest to out: assessment, HF, Navy, please, certification, flowered, direction, show,\n",
      "Nearest to most: identify, United, bank, Zen, Sigmund, apartments, budget, oa,\n",
      "Nearest to first: [134], safely, on, Leg, cause, -RRB#, waiting, acute,\n",
      "Nearest to into: on, in, for, ecological, of, and, adjective, first,\n",
      "Nearest to when: for, if, in, as, and, oa, cause, visit,\n",
      "Nearest to these: the, some, a, this, Snape, oa, complication, anniversary,\n",
      "Nearest to been: Odysseus, be, Guinea, were, incised, also, ion, endurance,\n",
      "Nearest to two: spiders, 0.00, shifting, Meaning, declared, measures, expedient, als,\n",
      "Nearest to found: oa, ulceration, Jacket, 1840s, trimester, neighbors, itchy, Puppies,\n",
      "Nearest to to: and, for, in,   , will, -RRB#, with, at,\n",
      "Nearest to years: exchanges, touch, learn, recourse, absent, circus, y, oa,\n",
      "Average loss at step  22000 :  7.262867607355118\n",
      "Average loss at step  24000 :  6.740637086749077\n",
      "Average loss at step  26000 :  6.319577817320823\n",
      "Average loss at step  28000 :  6.175960805892944\n",
      "Average loss at step  30000 :  5.779202470064163\n",
      "Nearest to your: the, his, a, or, their, irradiance, our, her,\n",
      "Nearest to would: can, may, must, will, should, was, also, Have,\n",
      "Nearest to at: for, in, and, from, to, defeat, -RRB#, by,\n",
      "Nearest to between: muscles, in, permanence, of, with, teens, or, rejoice,\n",
      "Nearest to like: carnivores, funeral, Video, and, after, hectares, 1850s, shared,\n",
      "Nearest to out: HF, Navy, assessment, flowered, please, certification, socks, saxifrage,\n",
      "Nearest to most: identify, United, bank, Zen, +32, Barbary, Intracellular, Honolulu,\n",
      "Nearest to first: [134], spaghetti, safely, drinkers, Leg, -RRB#, cause, through,\n",
      "Nearest to into: in, on, with, for, from, and, of, ecological,\n",
      "Nearest to when: for, if, and, as, in, oa, but, that,\n",
      "Nearest to these: the, some, many, this, complication, Snape, +32, oa,\n",
      "Nearest to been: were, was, Odysseus, be, incised, also, is, ion,\n",
      "Nearest to two: 0.00, spiders, Meaning, shifting, declared, expedient, als, -RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB*,\n",
      "Nearest to found: ulceration, 1840s, Puppies, neighbors, Jacket, trimester, +32, oa,\n",
      "Nearest to to: for,   , and, with, in, on, will, can,\n",
      "Nearest to years: exchanges, touch, -RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB*, recourse, +32, absent, refuted, learn,\n",
      "Average loss at step  32000 :  5.6665243488550185\n",
      "Average loss at step  34000 :  5.4315096833705905\n",
      "Average loss at step  36000 :  5.4413137454986575\n",
      "Average loss at step  38000 :  5.218340179562569\n",
      "Average loss at step  40000 :  5.223435809731483\n",
      "Nearest to your: his, the, their, my, our, a, her, irradiance,\n",
      "Nearest to would: can, may, will, must, should, might, was, parenteral,\n",
      "Nearest to at: in, for, from, and, -RRB#, defeat, by, with,\n",
      "Nearest to between: muscles, in, with, of, permanence, from, teens, plant’s,\n",
      "Nearest to like: carnivores, funeral, hectares, Video, after, -RCB-RCB-RCB-RCB-RCB-RCB, 1850s, shared,\n",
      "Nearest to out: HF, Navy, assessment, nonpolar, flowered, saxifrage, Café, socks,\n",
      "Nearest to most: United, identify, bank, Intracellular, Zen, Barbary, Honolulu, nomenclature,\n",
      "Nearest to first: [134], spaghetti, safely, drinkers, Leg, through, on, idea,\n",
      "Nearest to into: on, from, in, with, for, and, by, of,\n",
      "Nearest to when: if, for, as, in, but, where, oa, Parliament,\n",
      "Nearest to these: some, the, many, complication, this, Snape, other, +32,\n",
      "Nearest to been: were, was, be, Odysseus, incised, also, is, ion,\n",
      "Nearest to two: spiders, 0.00, expedient, Meaning, shifting, Figures, declared, grooming,\n",
      "Nearest to found: ulceration, Puppies, neighbors, trimester, +32, 1840s, oa, driven,\n",
      "Nearest to to: for, will, can,   , in, must, and, wishes,\n",
      "Nearest to years: exchanges, touch, absent, recourse, -RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB*, +32, days, refuted,\n",
      "Average loss at step  42000 :  5.10388218152523\n",
      "Average loss at step  44000 :  5.0777780500650405\n",
      "Average loss at step  46000 :  4.970336187839508\n",
      "Average loss at step  48000 :  4.951406133174896\n",
      "Average loss at step  50000 :  4.909587011814118\n",
      "Nearest to your: his, the, their, my, our, her, a, this,\n",
      "Nearest to would: will, may, can, must, should, could, might, to,\n",
      "Nearest to at: in, for, from, -RRB#, after, by, and, defeat,\n",
      "Nearest to between: muscles, permanence, in, with, from, of, giving, plant’s,\n",
      "Nearest to like: funeral, carnivores, as, and, after, shared, hectares, Video,\n",
      "Nearest to out: HF, assessment, Navy, flowered, nonpolar, Café, saxifrage, Distinguishing,\n",
      "Nearest to most: United, identify, more, Honolulu, Intracellular, bank, nomenclature, Zen,\n",
      "Nearest to first: spaghetti, [134], drinkers, colspan, idea, Leg, safely, through,\n",
      "Nearest to into: from, on, with, in, through, and, by, of,\n",
      "Nearest to when: if, for, as, where, but, in, Parliament, and,\n",
      "Nearest to these: some, the, many, other, complication, this, Kerala, Snape,\n",
      "Nearest to been: be, was, were, Odysseus, incised, gastropods, are, ion,\n",
      "Nearest to two: spiders, Meaning, 0.00, expedient, shifting, Figures, grooming, -RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB*,\n",
      "Nearest to found: ulceration, Puppies, neighbors, warlike, imperative, trimester, oa, +32,\n",
      "Nearest to to: for, will,   , can, would, wishes, -RRB#, and,\n",
      "Nearest to years: days, exchanges, touch, absent, recourse, -RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB*, +32, reiterated,\n",
      "Average loss at step  52000 :  4.866606753230095\n",
      "Average loss at step  54000 :  4.877653668284416\n",
      "Average loss at step  56000 :  4.803983922243118\n",
      "Average loss at step  58000 :  4.788582641005516\n",
      "Average loss at step  60000 :  4.772637385964393\n",
      "Nearest to your: his, their, the, my, our, her, a, this,\n",
      "Nearest to would: will, may, can, must, should, could, might, to,\n",
      "Nearest to at: in, for, -RRB#, from, after, defeat, oa, on,\n",
      "Nearest to between: in, muscles, permanence, with, from, of, giving, plant’s,\n",
      "Nearest to like: as, funeral, carnivores, king, shared, knows, hectares, ext,\n",
      "Nearest to out: assessment, Navy, HF, up, nonpolar, Distinguishing, flowered, saxifrage,\n",
      "Nearest to most: more, United, Honolulu, nomenclature, identify, Barbary, Intracellular, Zen,\n",
      "Nearest to first: spaghetti, [134], drinkers, idea, colspan, Leg, same, tongue,\n",
      "Nearest to into: from, through, on, in, with, by, Douglas-fir, acquire,\n",
      "Nearest to when: if, as, where, for, but, before, oa, in,\n",
      "Nearest to these: some, many, the, other, Some, all, Kerala, those,\n",
      "Nearest to been: be, was, were, Odysseus, is, become, incised, are,\n",
      "Nearest to two: spiders, Meaning, 0.00, expedient, Figures, grooming, three, shifting,\n",
      "Nearest to found: ulceration, Puppies, neighbors, warlike, imperative, boobies, trimester, mice,\n",
      "Nearest to to: for, will,   , -RRB#, can, would, wishes, must,\n",
      "Nearest to years: days, exchanges, weeks, absent, recourse, touch, months, -RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB*,\n",
      "Average loss at step  62000 :  4.751043798089027\n",
      "Average loss at step  64000 :  4.705071733236313\n",
      "Average loss at step  66000 :  4.683124627590179\n",
      "Average loss at step  68000 :  4.6702419724464415\n",
      "Average loss at step  70000 :  4.663221010923386\n",
      "Nearest to your: his, my, their, the, our, her, its, this,\n",
      "Nearest to would: will, may, can, must, should, could, might, parenteral,\n",
      "Nearest to at: in, for, -RRB#, after, from, on, oa, by,\n",
      "Nearest to between: in, muscles, from, permanence, with, giving, Draco, plant’s,\n",
      "Nearest to like: as, funeral, carnivores, hectares, ext, shared, and, knows,\n",
      "Nearest to out: assessment, up, HF, Navy, Distinguishing, nonpolar, flowered, saxifrage,\n",
      "Nearest to most: more, United, -RRB-LRBRGB, Intracellular, Barbary, Honolulu, nomenclature, bank,\n",
      "Nearest to first: spaghetti, [134], last, idea, drinkers, colspan, same, Leg,\n",
      "Nearest to into: from, through, in, on, with, by, and, carbonic,\n",
      "Nearest to when: if, where, before, but, in, as, oa, for,\n",
      "Nearest to these: some, many, the, Some, other, those, all, such,\n",
      "Nearest to been: be, were, was, become, Odysseus, by, already, gastropods,\n",
      "Nearest to two: spiders, Meaning, three, 0.00, expedient, Figures, grooming, McIntyre,\n",
      "Nearest to found: ulceration, Puppies, imperative, neighbors, warlike, -RRB-LRBRGB, boobies, oa,\n",
      "Nearest to to: for, will,   , -RRB#, wishes, can, and, would,\n",
      "Nearest to years: days, weeks, months, exchanges, absent, recourse, touch, -RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB-RCB*,\n",
      "Average loss at step  72000 :  4.681677001714706\n",
      "Average loss at step  74000 :  4.632661094844341\n",
      "Average loss at step  76000 :  4.639013104319573\n",
      "Average loss at step  78000 :  4.64439272904396\n",
      "Average loss at step  80000 :  4.590695064306259\n",
      "Nearest to your: his, my, their, our, the, her, its, a,\n",
      "Nearest to would: will, may, can, must, should, could, might, parenteral,\n",
      "Nearest to at: in, after, for, from, -RRB#, ++, on, by,\n",
      "Nearest to between: in, from, with, permanence, muscles, giving, Draco, around,\n",
      "Nearest to like: as, carnivores, funeral, ++, shared, oa, king, ext,\n",
      "Nearest to out: up, assessment, HF, Navy, Distinguishing, navel, off, saxifrage,\n",
      "Nearest to most: more, United, -RRB-LRBRGB, Honolulu, Intracellular, nomenclature, Marvels, Barbary,\n",
      "Nearest to first: last, spaghetti, [134], colspan, drinkers, idea, same, second,\n",
      "Nearest to into: through, from, on, in, with, Douglas-fir, by, over,\n",
      "Nearest to when: if, before, where, as, for, but, because, Parliament,\n",
      "Nearest to these: some, many, the, Some, those, such, other, all,\n",
      "Nearest to been: be, was, were, become, already, by, is, had,\n",
      "Nearest to two: three, Meaning, spiders, 0.00, several, Figures, four, many,\n",
      "Nearest to found: Puppies, ulceration, warlike, imperative, boobies, neighbors, -RRB-LRBRGB, said,\n",
      "Nearest to to: ++, for, will,   , To, can, -RRB#, -RCB-RCB-RCB-RCB-RCB-RCB,\n",
      "Nearest to years: days, weeks, months, absent, henna, exchanges, times, touch,\n",
      "Average loss at step  82000 :  4.594549582242966\n",
      "Average loss at step  84000 :  4.583482399344445\n",
      "Average loss at step  86000 :  4.576092893719673\n",
      "Average loss at step  88000 :  4.567452017307281\n",
      "Average loss at step  90000 :  4.567522046804428\n",
      "Nearest to your: his, my, their, our, her, the, its, this,\n",
      "Nearest to would: will, may, can, should, must, could, might, parenteral,\n",
      "Nearest to at: in, after, for, -RRB#, on, ++, from, under,\n",
      "Nearest to between: in, from, with, muscles, permanence, around, giving, Draco,\n",
      "Nearest to like: as, carnivores, ++, funeral, oa, king, ext, popular,\n",
      "Nearest to out: up, assessment, HF, Navy, off, Distinguishing, down, navel,\n",
      "Nearest to most: more, United, -RRB-LRBRGB, Marvels, Honolulu, nomenclature, Intracellular, Barbary,\n",
      "Nearest to first: last, spaghetti, [134], idea, colspan, second, drinkers, same,\n",
      "Nearest to into: through, from, on, with, in, by, Douglas-fir, surmounted,\n",
      "Nearest to when: if, before, where, after, but, as, because, oa,\n",
      "Nearest to these: some, many, Some, those, the, such, other, all,\n",
      "Nearest to been: be, was, were, become, already, had, acclimated, by,\n",
      "Nearest to two: three, Meaning, four, spiders, several, 0.00, five, Figures,\n",
      "Nearest to found: Puppies, ulceration, warlike, oa, boobies, -RRB-LRBRGB, ++, used,\n",
      "Nearest to to: To, for, will, ++, can,   , would, -RRB#,\n",
      "Nearest to years: days, weeks, months, absent, exchanges, times, henna, cases,\n",
      "Average loss at step  92000 :  4.560978431820869\n",
      "Average loss at step  94000 :  4.531102466702461\n",
      "Average loss at step  96000 :  4.544440076351166\n",
      "Average loss at step  98000 :  4.542137525916099\n",
      "Average loss at step  100000 :  4.520813816547394\n",
      "Nearest to your: my, his, their, our, her, the, its, a,\n",
      "Nearest to would: will, may, can, must, should, could, might, parenteral,\n",
      "Nearest to at: in, for, after, from, on, ++, -RRB#, under,\n",
      "Nearest to between: in, from, permanence, with, muscles, giving, Draco, around,\n",
      "Nearest to like: as, funeral, king, carnivores, ext, popular, ++, called,\n",
      "Nearest to out: up, off, assessment, down, HF, Distinguishing, Navy, navel,\n",
      "Nearest to most: more, Marvels, United, -RRB-LRBRGB, some, nomenclature, Honolulu, naive,\n",
      "Nearest to first: last, [134], spaghetti, colspan, second, same, drinkers, idea,\n",
      "Nearest to into: through, from, in, on, with, Douglas-fir, surmounted, and,\n",
      "Nearest to when: if, before, where, but, because, after, until, as,\n",
      "Nearest to these: some, many, Some, those, such, the, all, other,\n",
      "Nearest to been: be, was, were, become, already, had, acclimated, by,\n",
      "Nearest to two: three, four, several, Meaning, spiders, five, many, 0.00,\n",
      "Nearest to found: Puppies, ulceration, used, warlike, boobies, said, imperative, -RRB-LRBRGB,\n",
      "Nearest to to: for, ++, To, will, -RRB#, adherent, must, -RCB-RCB-RCB-RCB-RCB-RCB,\n",
      "Nearest to years: days, weeks, months, times, absent, henna, exchanges, cases,\n"
     ]
    }
   ],
   "source": [
    "# Step 5: Begin training.\n",
    "num_steps = 100001\n",
    "\n",
    "with tf.Session(graph = graph) as session:\n",
    "  # We must initialize all variables before we use them.\n",
    "  init.run()\n",
    "  print(\"Initialized\")\n",
    "\n",
    "  average_loss = 0\n",
    "  for step in xrange(num_steps):\n",
    "    batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)\n",
    "    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "\n",
    "    # We perform one update step by evaluating the optimizer op (including it in the list of returned values for session.run()\n",
    "    _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "    average_loss += loss_val\n",
    "\n",
    "    if step % 2000 == 0:\n",
    "      if step > 0:\n",
    "        average_loss /= 2000\n",
    "      # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "      print(\"Average loss at step \", step, \": \", average_loss)\n",
    "      average_loss = 0\n",
    "\n",
    "    # Note that this is expensive (~20% slowdown if computed every 500 steps)\n",
    "    if step % 10000 == 0:\n",
    "      sim = similarity.eval()\n",
    "      for i in xrange(valid_size):\n",
    "        valid_word = reverse_dictionary[valid_examples[i]]\n",
    "        top_k = 8  # number of nearest neighbors\n",
    "        nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "        log_str = \"Nearest to %s:\" % valid_word # finding the k nearest neighbors to the validation word\n",
    "        for k in xrange(top_k):\n",
    "          close_word = reverse_dictionary[nearest[k]]\n",
    "          log_str = \"%s %s,\" % (log_str, close_word)\n",
    "        print(log_str)\n",
    "\n",
    "  # Get final embeddings after training is complete\n",
    "  final_embeddings = normalized_embeddings.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 128)\n"
     ]
    }
   ],
   "source": [
    "print(final_embeddings.shape) # from building the graph, embedding weights are vocabulary_size x embedding_size matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocabulary_size: 50000\n",
      "embedding_size: 128\n"
     ]
    }
   ],
   "source": [
    "print(\"vocabulary_size:\", vocabulary_size)\n",
    "embedding_size = final_embeddings.shape[1]\n",
    "print(\"embedding_size:\", embedding_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(\n",
    "  fname = \"vocab.tsv\", \n",
    "  X = np.array(object = list(dictionary.keys())), \n",
    "  fmt = '%s', \n",
    "  delimiter = '\\n') # write vocabulary to file\n",
    "np.savetxt(\n",
    "  fname = \"word_embeddings.csv\", \n",
    "  X = final_embeddings, \n",
    "  delimiter = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UNK\n",
      "<s>\n",
      "</s>\n",
      "the\n",
      "of\n",
      "and\n",
      "to\n",
      "a\n",
      "in\n",
      "is\n"
     ]
    }
   ],
   "source": [
    "!head -10 vocab.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !head -5 word_embeddings.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read in trained embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pre_trained_embeddings_array = np.loadtxt(\n",
    "  fname = \"word_embeddings.csv\", \n",
    "  dtype = np.float64, \n",
    "  delimiter = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocabulary_size = 50000 & embedding_size = 128\n"
     ]
    }
   ],
   "source": [
    "vocabulary_size = pre_trained_embeddings_array.shape[0]\n",
    "embedding_size = pre_trained_embeddings_array.shape[1]\n",
    "print(\"vocabulary_size = {} & embedding_size = {}\".format(vocabulary_size, embedding_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create TensorFlow Dual LSTM Question Answering Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "UNKNOWN_TOKEN = 0\n",
    "START_TOKEN = 1\n",
    "STOP_TOKEN = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "arguments = {}\n",
    "# File arguments\n",
    "arguments[\"train_file_pattern\"] = \"train_data.csv\"\n",
    "arguments[\"eval_file_pattern\"] = \"eval_data.csv\"\n",
    "arguments[\"output_dir\"] = \"trained_model\"\n",
    "\n",
    "# Sequence shape hyperparameters\n",
    "arguments[\"batch_size\"] = 16\n",
    "arguments[\"reverse_feature_sequence\"] = True\n",
    "\n",
    "# LSTM hyperparameters\n",
    "arguments[\"question_lstm_hidden_units\"] = [256, 128, 64]\n",
    "arguments[\"question_lstm_dropout_output_keep_probs\"] = [0.99, 0.97, 0.95]\n",
    "arguments[\"choices_lstm_hidden_units\"] = [256, 128, 64]\n",
    "arguments[\"choices_lstm_dropout_output_keep_probs\"] = [0.99, 0.97, 0.95]\n",
    "\n",
    "# DNN hyperparameters\n",
    "arguments[\"use_question_dnn\"] = True\n",
    "arguments[\"question_dnn_hidden_units\"] = [64, 32, 16]\n",
    "arguments[\"question_dnn_dropout_rates\"] = [0.01, 0.03, 0.05]\n",
    "arguments[\"use_choices_dnn\"] = True\n",
    "arguments[\"choices_dnn_hidden_units\"] = [64, 32, 16]\n",
    "arguments[\"choices_dnn_dropout_rates\"] = [0.01, 0.03, 0.05]\n",
    "arguments[\"final_logits_size\"] = 16\n",
    "\n",
    "# Training parameters\n",
    "arguments[\"train_steps\"] = 1000\n",
    "arguments[\"learning_rate\"] = 0.1\n",
    "arguments[\"start_delay_secs\"] = 30\n",
    "arguments[\"throttle_secs\"] = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set logging to be level of INFO\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine CSV and label columns\n",
    "CSV_COLUMNS = \"question,choices,answer_idx\".split(',')\n",
    "LABEL_COLUMN = \"answer_idx\"\n",
    "VOCAB_FILE_PATH = \"vocab.tsv\" # where vocabulary is saved, dynamically set in train_and_eval function\n",
    "PADWORD = 'ZYXW'\n",
    "\n",
    "# Set default values for each CSV column\n",
    "DEFAULTS = [[\"\"], [\"\"], [0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an input function reading a file using the Dataset API\n",
    "# Then provide the results to the Estimator API\n",
    "def read_dataset(filename, mode, batch_size, params):\n",
    "  def _input_fn():\n",
    "    def decode_csv(value_column):\n",
    "      columns = tf.decode_csv(\n",
    "        records = value_column, record_defaults = DEFAULTS, field_delim = ',')\n",
    "      features = dict(zip(CSV_COLUMNS, columns))\n",
    "      labels = tf.cast(x = features.pop(LABEL_COLUMN), dtype = tf.int64)\n",
    "      \n",
    "      return features, labels\n",
    "    \n",
    "    # Create list of files that match pattern\n",
    "    file_list = tf.gfile.Glob(filename = filename)\n",
    "\n",
    "    # Create dataset from file list\n",
    "    dataset = tf.data.TextLineDataset(filenames = file_list)  # Read text file\n",
    "\n",
    "    # Decode the CSV file into a features dictionary of tensors\n",
    "    dataset = dataset.map(map_func = decode_csv)\n",
    "    \n",
    "    # Determine amount of times to repeat file based on if we are training or evaluating\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      num_epochs = None # indefinitely\n",
    "    else:\n",
    "      num_epochs = 1 # end-of-input after this\n",
    "\n",
    "    # Repeat files num_epoch times\n",
    "    dataset = dataset.repeat(count = num_epochs)\n",
    "\n",
    "    # Group the data into batches\n",
    "    dataset = dataset.batch(batch_size = batch_size)\n",
    "    \n",
    "    # Determine if we should shuffle based on if we are training or evaluating\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      dataset = dataset.shuffle(buffer_size = 10 * batch_size)\n",
    "\n",
    "    # Create a iterator and then pull the next batch of features from the example queue\n",
    "    batch_features, batch_labels = dataset.make_one_shot_iterator().get_next()\n",
    "\n",
    "    return batch_features, batch_labels\n",
    "  return _input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create our model function to be used in our custom estimator\n",
    "def dual_lstm_text_question_answer(features, labels, mode, params):\n",
    "  # Function to split string tensors into multiple substring tensors\n",
    "  def split_strings(features):\n",
    "    # Function to split string tensors into substring tensors\n",
    "    def split_string_into_substrings(string_tensor, delimiter):\n",
    "      # Split string tensor into a sparse tensor based on delimiter\n",
    "      # indices: shape = (cur_batch_size, 2), values: shape = (cur_batch_size,), dense_shape: shape = (2,)\n",
    "      split_string_sparse_tensor = tf.string_split(\n",
    "        source = string_tensor, \n",
    "        delimiter = delimiter)\n",
    "\n",
    "      # Create a dense tensor of the float values that were converted from text csv\n",
    "      # shape = (cur_batch_size, max_substrings_across_batch)\n",
    "      split_string_dense_tensor = tf.sparse_tensor_to_dense(\n",
    "        sp_input = split_string_sparse_tensor, \n",
    "        default_value = PADWORD)\n",
    "\n",
    "      # The index for the sequence I am currently on, the first column of the sparse tensor indicies\n",
    "      # shape = (cur_batch_size,)\n",
    "      sequence_index = split_string_sparse_tensor.indices[:, 0]\n",
    "\n",
    "      # The index for the timestep I am currently on, the second column of the sparse tensor indicies\n",
    "      # shape = (cur_batch_size,)\n",
    "      time_index = split_string_sparse_tensor.indices[:, 1]\n",
    "\n",
    "      # The sequence lengths for each sequence\n",
    "      # shape = (cur_batch_size,)\n",
    "      lengths_vector = tf.segment_max(\n",
    "        data = time_index, \n",
    "        segment_ids = sequence_index) + 1\n",
    "\n",
    "      return split_string_dense_tensor, lengths_vector\n",
    "\n",
    "    # Function to split a single string of multiple setences into words\n",
    "    def split_multi_sentence_string_into_words(multi_sentence_string_tensor):\n",
    "      # split_multi_sentence_string_dense_tensor.shape = (cur_batch_size, max_num_choices_across_batch)\n",
    "      # num_sentences_vector.shape = (cur_batch_size,)\n",
    "      split_multi_sentence_string_dense_tensor, num_sentences_vector = \\\n",
    "        split_string_into_substrings(string_tensor = multi_sentence_string_tensor, delimiter = ';')\n",
    "\n",
    "      cur_batch_size = tf.shape(\n",
    "        input = split_multi_sentence_string_dense_tensor, \n",
    "        out_type = tf.int64)[0] # shape = ()\n",
    "\n",
    "      # Calculate the max number of choices across all questions in the batch\n",
    "      max_num_choices_across_batch = tf.reduce_max(\n",
    "        input_tensor = num_sentences_vector) # shape = ()\n",
    "\n",
    "      split_multi_sentence_string_dense_tensor_flattened = tf.reshape(\n",
    "        tensor = split_multi_sentence_string_dense_tensor, \n",
    "        shape = [cur_batch_size * max_num_choices_across_batch])\n",
    "\n",
    "      # split_multi_sentence_word_string_dense_tensor_flattened.shape = (cur_batch_size * max_num_choices_across_batch, max_num_words_across_batch)\n",
    "      # num_multi_sentence_word_tensor_flattened.shape = (cur_batch_size * max_num_choices_across_batch,)\n",
    "      split_multi_sentence_word_string_dense_tensor_flattened, num_multi_sentence_word_tensor_flattened = \\\n",
    "        split_string_into_substrings(string_tensor = split_multi_sentence_string_dense_tensor_flattened, delimiter = ' ')\n",
    "\n",
    "      max_num_words_across_batch = tf.reduce_max(\n",
    "        input_tensor = num_multi_sentence_word_tensor_flattened) # shape = ()\n",
    "\n",
    "      split_multi_sentence_word_string_dense_tensor = tf.reshape(\n",
    "        tensor = split_multi_sentence_word_string_dense_tensor_flattened, \n",
    "        shape = [cur_batch_size, max_num_choices_across_batch, max_num_words_across_batch])\n",
    "\n",
    "      num_multi_sentence_word_tensor = tf.reshape(\n",
    "        tensor = num_multi_sentence_word_tensor_flattened, \n",
    "        shape = [cur_batch_size, max_num_choices_across_batch])\n",
    "\n",
    "      return split_multi_sentence_word_string_dense_tensor, num_multi_sentence_word_tensor, split_multi_sentence_string_dense_tensor, num_sentences_vector\n",
    "\n",
    "    # Start split_strings function\n",
    "    # question_split_words_strings_tensor.shape = (cur_batch_size, max_num_question_words_across_batch)\n",
    "    # question_num_words.shape = (cur_batch_size,)\n",
    "    question_split_words_strings_tensor, question_num_words = \\\n",
    "      split_string_into_substrings(string_tensor = features[\"question\"], delimiter = ' ')\n",
    "\n",
    "    # choices_split_words_strings_tensor.shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch)\n",
    "    # choices_num_words.shape = (cur_batch_size, max_num_choices)\n",
    "    # choices_split_sentences_strings_tensor.shape = (cur_batch_size, max_num_choices)\n",
    "    # choices_num_sentences. shape = (cur_batch_size,)\n",
    "    choices_split_words_strings_tensor, choices_num_words, choices_split_sentences_strings_tensor, choices_num_sentences = \\\n",
    "      split_multi_sentence_string_into_words(features[\"choices\"])\n",
    "\n",
    "    return question_split_words_strings_tensor, question_num_words, choices_split_words_strings_tensor, choices_num_words, choices_split_sentences_strings_tensor, choices_num_sentences\n",
    "\n",
    "  # question_split_words_strings_tensor.shape = (cur_batch_size, max_num_question_words_across_batch)\n",
    "  # question_num_words.shape = (cur_batch_size,)\n",
    "  # choices_split_words_strings_tensor.shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch)\n",
    "  # choices_num_words.shape = (cur_batch_size, max_num_choices)\n",
    "  # choices_split_sentences_strings_tensor.shape = (cur_batch_size, max_num_choices)\n",
    "  # choices_num_sentences.shape = (cur_batch_size,)\n",
    "  question_split_words_strings_tensor, question_num_words, choices_split_words_strings_tensor, choices_num_words, choices_split_sentences_strings_tensor, choices_num_sentences = \\\n",
    "    split_strings(features)\n",
    "  \n",
    "  # Map each word to respective integer index\n",
    "  word_to_id_lookup_table = tf.contrib.lookup.index_table_from_file(\n",
    "    vocabulary_file = \"vocab.tsv\",\n",
    "    num_oov_buckets = 0,\n",
    "    vocab_size = vocabulary_size,\n",
    "    default_value = 0,  # for words not in vocabulary (OOV)\n",
    "    delimiter = ' ')\n",
    "\n",
    "  # shape = (cur_batch_size, max_num_question_words_across_batch)\n",
    "  question_split_words_ids_tensor = word_to_id_lookup_table.lookup(\n",
    "    question_split_words_strings_tensor)\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch)\n",
    "  choices_split_words_ids_tensor = word_to_id_lookup_table.lookup(\n",
    "    choices_split_words_strings_tensor)\n",
    "    \n",
    "  # Load trained embeddings into variable\n",
    "  # shape = (vocabulary_size, embedding_size)\n",
    "  embeddings_placeholder = tf.placeholder(\n",
    "    dtype = tf.float64, \n",
    "    shape = [vocabulary_size, embedding_size], \n",
    "    name = \"embedding_placeholder\")\n",
    "  \n",
    "  # shape = (vocabulary_size, embedding_size)\n",
    "  embeddings_variable = tf.Variable(\n",
    "    initial_value = embeddings_placeholder, \n",
    "    trainable = True, \n",
    "    name = \"embedding_variable\", \n",
    "    dtype = tf.float64, \n",
    "    expected_shape = [vocabulary_size, embedding_size])\n",
    "\n",
    "  # Get dynamic batch size in case there was a partially filled batch\n",
    "  cur_batch_size32 = tf.shape(\n",
    "    input = question_split_words_ids_tensor, \n",
    "    out_type = tf.int32)[0] # shape = ()\n",
    "  \n",
    "  cur_batch_size64 = tf.shape(\n",
    "    input = question_split_words_ids_tensor, \n",
    "    out_type = tf.int64)[0] # shape = ()\n",
    "  \n",
    "  # Calculate the max number of choices across all questions in the batch\n",
    "  max_num_choices_across_batch = tf.reduce_max(\n",
    "    input_tensor = choices_num_sentences) # shape = ()\n",
    "  \n",
    "  max_num_choice_words_across_batch = tf.reduce_max(\n",
    "    input_tensor = choices_num_words) # shape = ()\n",
    "\n",
    "  # Gather the embedding vectors for the question's words\n",
    "  # shape = (cur_batch_size, max_num_question_words_across_batch, embedding_size)\n",
    "  question_split_words_embeddings_tensor = tf.nn.embedding_lookup(\n",
    "    params = embeddings_variable, \n",
    "    ids = question_split_words_ids_tensor)\n",
    "  \n",
    "  # Gather the embedding vectors for the answer's words\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch, embedding_size)\n",
    "  choices_split_words_embeddings_tensor = tf.nn.embedding_lookup(\n",
    "    params = embeddings_variable, \n",
    "    ids = choices_split_words_ids_tensor)\n",
    "  \n",
    "  # It usually works better to have the features in reverse order due to higher correlation between the later feature timesteps and earlier label timesteps\n",
    "  if params[\"reverse_feature_sequence\"] == True:\n",
    "    # shape = (cur_batch_size, max_num_question_words_across_batch, embedding_size)\n",
    "    question_split_words_embeddings_tensor = tf.reverse_sequence(\n",
    "      input = question_split_words_embeddings_tensor, \n",
    "      seq_lengths = question_num_words, \n",
    "      seq_axis = 1, \n",
    "      batch_axis = 0)\n",
    "\n",
    "  ################################################################################\n",
    "  \n",
    "  # Function to create stack of LSTM\n",
    "  def create_LSTM_stack(lstm_hidden_units, lstm_dropout_output_keep_probs):\n",
    "    # First create a list of LSTM cells using our list of lstm hidden unit sizes\n",
    "    lstm_cells = [tf.nn.rnn_cell.BasicLSTMCell(\n",
    "      num_units = units, \n",
    "      state_is_tuple = True) \n",
    "                  for units in lstm_hidden_units] # list of LSTM cells\n",
    "\n",
    "    # Next apply a dropout wrapper to our stack of LSTM cells, in this case just on the outputs\n",
    "    dropout_lstm_cells = [tf.nn.rnn_cell.DropoutWrapper(\n",
    "      cell = lstm_cells[cell_index], \n",
    "      input_keep_prob = 1.0, \n",
    "      output_keep_prob = lstm_dropout_output_keep_probs[cell_index], \n",
    "      state_keep_prob = 1.0) \n",
    "                          for cell_index in range(len(lstm_cells))]\n",
    "\n",
    "    # Create a stack of layers of LSTM cells\n",
    "    stacked_lstm_cells = tf.nn.rnn_cell.MultiRNNCell(\n",
    "      cells = dropout_lstm_cells, \n",
    "      state_is_tuple = True) # combines list into MultiRNNCell object\n",
    "\n",
    "    return stacked_lstm_cells\n",
    "  \n",
    "  ################################################################################\n",
    "\n",
    "  # Create our question LSTM stack now\n",
    "  question_stacked_lstm_cells = create_LSTM_stack(\n",
    "    lstm_hidden_units = params[\"question_lstm_hidden_units\"], \n",
    "    lstm_dropout_output_keep_probs = params[\"question_lstm_dropout_output_keep_probs\"])\n",
    "\n",
    "  # Encode the question embedding sequence using our question stack of LSTMs\n",
    "  # question_outputs.shape = (cur_batch_size, max_question_num_words, question_lstm_hidden_units[-1])\n",
    "  # question_states tuple of question final c_state and h_state for each layer\n",
    "  question_outputs, question_states = tf.nn.dynamic_rnn(\n",
    "    cell = question_stacked_lstm_cells, \n",
    "    inputs = question_split_words_embeddings_tensor,\n",
    "    sequence_length = question_num_words,\n",
    "    initial_state = question_stacked_lstm_cells.zero_state(\n",
    "      batch_size = cur_batch_size32, \n",
    "      dtype = tf.float64), \n",
    "    dtype = tf.float64)\n",
    "\n",
    "  # Extract the question final output\n",
    "  # shape = (cur_batch_size, question_lstm_hidden_units[-1])\n",
    "  question_final_outputs = tf.map_fn(\n",
    "    fn = lambda x: question_outputs[x[0], x[1], :], \n",
    "    elems = (tf.range(start = 0, limit = cur_batch_size64, dtype = tf.int64), \n",
    "             question_num_words - 1), \n",
    "    dtype = tf.float64)\n",
    "  \n",
    "  ################################################################################\n",
    "  \n",
    "  # Create the input layer to our question DNN\n",
    "  # shape = (cur_batch_size, question_lstm_hidden_units[-1])\n",
    "  question_dnn_network = question_final_outputs\n",
    "  \n",
    "  if params[\"use_question_dnn\"] == True:\n",
    "    # Add hidden layers with the given number of units/neurons per question DNN layer\n",
    "    for i in range(0, len(params['question_dnn_hidden_units'])):\n",
    "      # shape = (cur_batch_size, question_dnn_hidden_units[i])\n",
    "      question_dnn_network = tf.layers.dense(\n",
    "        inputs = question_dnn_network, \n",
    "        units = params['question_dnn_hidden_units'][i], \n",
    "        activation = tf.nn.relu)\n",
    "      question_dnn_network = tf.layers.dropout(\n",
    "        inputs = question_dnn_network, \n",
    "        rate = params['question_dnn_dropout_rates'][i])\n",
    "  \n",
    "  # Add question final logits layer\n",
    "  # shape = (cur_batch_size, final_logits_size)\n",
    "  question_logits = tf.layers.dense(\n",
    "    inputs = question_dnn_network, \n",
    "    units = params[\"final_logits_size\"], \n",
    "    activation = None)\n",
    "\n",
    "  ################################################################################\n",
    "  \n",
    "   # Create our choices LSTM stack now\n",
    "  choices_stacked_lstm_cells = create_LSTM_stack(\n",
    "    lstm_hidden_units = params[\"choices_lstm_hidden_units\"], \n",
    "    lstm_dropout_output_keep_probs = params[\"choices_lstm_dropout_output_keep_probs\"])\n",
    "  \n",
    "  # Encode the choices embedding sequence using our choices stack of LSTMs in while loop\n",
    "  def while_loop_condition(choice_index, max_num_choices_across_batch, choices_split_words_embeddings_tensor, choices_num_words, cur_batch_size32, choices_outputs):\n",
    "    # shape = ()\n",
    "    condition = tf.less(\n",
    "      x = choice_index, \n",
    "      y = max_num_choices_across_batch)\n",
    "\n",
    "    return condition\n",
    "\n",
    "  def while_loop_body(choice_index, max_num_choices_across_batch, choices_split_words_embeddings_tensor, choices_num_words, cur_batch_size32, choices_outputs):\n",
    "    with tf.variable_scope(\n",
    "      name_or_scope = \"choices_lstm_stack\", reuse = tf.AUTO_REUSE):\n",
    "      # Run each choice through the choices LSTM stack\n",
    "      # shape = (cur_batch_size, max_num_choices_words_across_batch, choices_lstm_hidden_units[-1])\n",
    "      choice_outputs, _ = tf.nn.dynamic_rnn(\n",
    "        cell = choices_stacked_lstm_cells, \n",
    "        inputs = choices_split_words_embeddings_tensor[:, choice_index, :, :],\n",
    "        sequence_length = choices_num_words[:, choice_index],\n",
    "        initial_state = choices_stacked_lstm_cells.zero_state(\n",
    "          batch_size = cur_batch_size32, \n",
    "          dtype = tf.float64), \n",
    "        dtype = tf.float64)\n",
    "\n",
    "    # Concatenate the outputs into choice collection\n",
    "    # shape = (cur_batch_size, choice_index + 1, max_num_choices_words_across_batch, choices_lstm_hidden_units[-1])\n",
    "    choices_outputs = tf.concat(\n",
    "      values = [choices_outputs, tf.expand_dims(input = choice_outputs, axis = 1)], \n",
    "      axis = 1)\n",
    "\n",
    "    # Increment to the next choice\n",
    "    choice_index += 1\n",
    "    \n",
    "    return choice_index, max_num_choices_across_batch, choices_split_words_embeddings_tensor, choices_num_words, cur_batch_size32, choices_outputs\n",
    "\n",
    "  # Initial loop variable and output tensor\n",
    "  choice_index = tf.constant(value = 0, dtype = tf.int64) # shape = ()\n",
    "  # shape = (cur_batch_size, 0, max_num_choices_words_across_batch, choices_lstm_hidden_units[-1])\n",
    "  choices_outputs = tf.zeros(\n",
    "    shape = [tf.cast(\n",
    "      x = cur_batch_size32, \n",
    "      dtype = tf.int64), \n",
    "             0, \n",
    "             max_num_choice_words_across_batch, \n",
    "             params[\"choices_lstm_hidden_units\"][-1]], \n",
    "    dtype = tf.float64)\n",
    "\n",
    "  # Call the while loop\n",
    "  # choices_outputs.shape = (cur_batch_size, max_num_choices_across_batch, max_num_choices_words_across_batch, choices_lstm_hidden_units[-1])\n",
    "  choice_index, max_num_choices_across_batch, choices_split_words_embeddings_tensor, choices_num_words, cur_batch_size32, choices_outputs = \\\n",
    "    tf.while_loop(\n",
    "      cond = while_loop_condition, \n",
    "      body = while_loop_body, \n",
    "      loop_vars = [choice_index, max_num_choices_across_batch, choices_split_words_embeddings_tensor, choices_num_words, cur_batch_size32, choices_outputs],\n",
    "      shape_invariants = [\n",
    "        choice_index.get_shape(), \n",
    "        max_num_choices_across_batch.get_shape(),\n",
    "        choices_split_words_embeddings_tensor.get_shape(), \n",
    "        choices_num_words.get_shape(), \n",
    "        cur_batch_size32.get_shape(), \n",
    "        tf.TensorShape(dims = [None, None, None, params[\"choices_lstm_hidden_units\"][-1]])\n",
    "      ])\n",
    "\n",
    "  # Extract the choices final output\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch)\n",
    "  tiled_batch_indices = tf.map_fn(\n",
    "    fn = lambda x: tf.tile(\n",
    "      input = [x], \n",
    "      multiples = [max_num_choices_across_batch]), \n",
    "    elems = tf.range(start = 0, limit = cur_batch_size64, dtype = tf.int64), \n",
    "    dtype = tf.int64)\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch)\n",
    "  tiled_batch_indices_flattened = tf.reshape(\n",
    "    tensor = tiled_batch_indices, \n",
    "    shape = [cur_batch_size64 * max_num_choices_across_batch])\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch)\n",
    "  tiled_choice_indices = tf.tile(\n",
    "    input = tf.range(start = 0, limit = max_num_choices_across_batch, dtype = tf.int64), \n",
    "    multiples = [cur_batch_size64])\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch)\n",
    "  choices_num_words_flattened = tf.reshape(\n",
    "    tensor = choices_num_words, \n",
    "    shape = [cur_batch_size64 * max_num_choices_across_batch])\n",
    "\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch, choices_lstm_hidden_units[-1])\n",
    "  choices_final_outputs_flat = tf.map_fn(\n",
    "    fn = lambda x: choices_outputs[x[0], x[1], x[2], :], \n",
    "    elems = (tiled_batch_indices_flattened, tiled_choice_indices, choices_num_words_flattened - 1), \n",
    "    dtype = tf.float64)\n",
    "\n",
    "  ################################################################################\n",
    "\n",
    "  # Create the input layer to our choices DNN\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch, choices_lstm_hidden_units[-1])\n",
    "  choices_dnn_network = choices_final_outputs_flat\n",
    "  \n",
    "  if params[\"use_choices_dnn\"] == True:\n",
    "    # Add hidden layers with the given number of units/neurons per question DNN layer\n",
    "    for i in range(0, len(params['choices_dnn_hidden_units'])):\n",
    "      # shape = (cur_batch_size * max_num_choices_across_batch, choices_dnn_hidden_units[i])\n",
    "      choices_dnn_network = tf.layers.dense(\n",
    "        inputs = choices_dnn_network, \n",
    "        units = params['choices_dnn_hidden_units'][i], \n",
    "        activation = tf.nn.relu)\n",
    "      choices_dnn_network = tf.layers.dropout(\n",
    "        inputs = choices_dnn_network, \n",
    "        rate = params['choices_dnn_dropout_rates'][i])\n",
    "  \n",
    "  # Add choices final logits layer\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch, final_logits_size)\n",
    "  choices_logits_flat = tf.layers.dense(\n",
    "    inputs = choices_dnn_network, \n",
    "    units = params[\"final_logits_size\"], \n",
    "    activation = None)\n",
    "\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch, final_logits_size)\n",
    "  choices_logits = tf.reshape(\n",
    "    tensor = choices_logits_flat, \n",
    "    shape = [cur_batch_size64, max_num_choices_across_batch, params[\"final_logits_size\"]])\n",
    "  \n",
    "  ################################################################################\n",
    "  \n",
    "  # Loss function, training/eval ops\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "    # shape = (cur_batch_size, final_logits_size)\n",
    "    answer_logits = tf.map_fn(\n",
    "      fn = lambda x: choices_logits[x[0], x[1], :], \n",
    "      elems = (tf.range(start = 0, limit = cur_batch_size64, dtype = tf.int64), labels), \n",
    "      dtype = tf.float64)\n",
    "    \n",
    "    predictions_dict = None\n",
    "    loss = tf.losses.mean_squared_error(\n",
    "      labels = answer_logits, predictions = question_logits)\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = params['learning_rate'],\n",
    "      optimizer = \"Adam\")\n",
    "    eval_metric_ops = None\n",
    "    export_outputs = None\n",
    "  else:\n",
    "    # Normalize logits\n",
    "    # shape = (cur_batch_size, final_logits_size)\n",
    "    question_logits_normalized = tf.nn.l2_normalize(\n",
    "      x = question_logits, axis = 1)\n",
    "    # shape = (cur_batch_size, max_num_choices_across_batch, final_logits_size)\n",
    "    choices_logits_normalized = tf.nn.l2_normalize(\n",
    "      choices_logits, axis = 2)\n",
    "\n",
    "    # Find cosine similiarites\n",
    "    # shape = (cur_batch_size, max_num_choices_across_batch)\n",
    "    cosine_similarities = tf.transpose(\n",
    "      a = tf.map_fn(\n",
    "        fn = lambda x: tf.reduce_sum(\n",
    "          input_tensor = tf.multiply(\n",
    "            x = question_logits_normalized, \n",
    "            y = choices_logits_normalized[:, x, :]), \n",
    "          axis = 1), \n",
    "        elems = tf.range(start = 0, limit = max_num_choices_across_batch, dtype = tf.int64), \n",
    "        dtype = tf.float64))\n",
    "    \n",
    "    # Find euclidean distances\n",
    "    # shape = (cur_batch_size, max_num_choices_across_batch)\n",
    "    euclidean_distances = tf.transpose(\n",
    "      a = tf.map_fn(\n",
    "        fn = lambda x: tf.norm(\n",
    "          tensor = question_logits - choices_logits[:, x, :], \n",
    "          axis = 1), \n",
    "        elems = tf.range(start = 0, limit = max_num_choices_across_batch, dtype = tf.int64), \n",
    "        dtype = tf.float64))\n",
    "\n",
    "#     # Choose the index with the highest cosine similarity for each element in the batch\n",
    "#     predicted_answer_index = tf.argmax(input = cosine_similarities, axis = 1) # shape = (cur_batch_size,)\n",
    "\n",
    "    # Choose the index with the lowest euclidean distance for each element in the batch\n",
    "    # shape = (cur_batch_size,)\n",
    "    predicted_answer_index = tf.map_fn(\n",
    "      fn = lambda x: tf.argmin(\n",
    "        input = euclidean_distances[x[0], 0:x[1]], \n",
    "        axis = 0, \n",
    "        output_type = tf.int64), \n",
    "      elems = (tf.range(start = 0, limit = cur_batch_size64, dtype = tf.int64), \n",
    "               choices_num_sentences), \n",
    "      dtype = tf.int64)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.EVAL:\n",
    "      # shape = (cur_batch_size, final_logits_size)\n",
    "      answer_logits = tf.map_fn(\n",
    "        fn = lambda x: choices_logits[x[0], x[1], :], \n",
    "        elems = (tf.range(start = 0, limit = cur_batch_size64, dtype = tf.int64), labels), \n",
    "        dtype = tf.float64)\n",
    "      \n",
    "      predictions_dict = None\n",
    "      loss = tf.losses.mean_squared_error(\n",
    "        labels = answer_logits, predictions = question_logits)\n",
    "      train_op = None\n",
    "      eval_metric_ops = {\n",
    "        \"rmse\": tf.metrics.root_mean_squared_error(\n",
    "          labels = answer_logits, predictions = question_logits),\n",
    "        \"accuracy\": tf.metrics.accuracy(\n",
    "          labels = labels, predictions = predicted_answer_index),\n",
    "        \"mean_cosine_distance\": tf.metrics.mean_cosine_distance(\n",
    "          labels = tf.nn.l2_normalize(x = answer_logits, axis = 1), \n",
    "          predictions = tf.nn.l2_normalize(x = question_logits, axis = 1), \n",
    "          dim = 1)\n",
    "      }\n",
    "      export_outputs = None\n",
    "    else:\n",
    "      # shape = (cur_batch_size)\n",
    "      predicted_answer_text = tf.gather_nd(\n",
    "        params = choices_split_sentences_strings_tensor, \n",
    "        indices = tf.stack(\n",
    "          values = [tf.range(start = 0, limit = cur_batch_size64, dtype = tf.int64), \n",
    "                    predicted_answer_index], \n",
    "          axis = 1))\n",
    "      \n",
    "      # Create predictions\n",
    "      predictions_dict = {\n",
    "        \"cosine_similarities\": cosine_similarities, \n",
    "        \"euclidean_distances\": euclidean_distances, \n",
    "        \"predicted_answer_index\": predicted_answer_index, \n",
    "        \"question_text\": features[\"question\"],\n",
    "        \"predicted_answer_text\": predicted_answer_text,\n",
    "        \"question_logits\": question_logits,\n",
    "        \"choices_logits\": choices_logits,\n",
    "        \"choices_num_sentences\": choices_num_sentences,\n",
    "        \"choices_num_words\": choices_num_words,\n",
    "        \"features['choices']\": features[\"choices\"],\n",
    "        \"max_num_choices_across_batch\": tf.tile(\n",
    "          input = [max_num_choices_across_batch], multiples = [cur_batch_size64]),\n",
    "        \"max_num_choice_words_across_batch\": tf.tile(\n",
    "          input = [max_num_choice_words_across_batch], multiples = [cur_batch_size64]),\n",
    "        \"choices_split_words_strings_tensor\": choices_split_words_strings_tensor,\n",
    "        \"choices_split_sentences_strings_tensor\": choices_split_sentences_strings_tensor,\n",
    "        \"choices_final_outputs_flat\": tf.reshape(\n",
    "          tensor = choices_final_outputs_flat, \n",
    "          shape = [cur_batch_size64, max_num_choices_across_batch, params[\"choices_lstm_hidden_units\"][-1]]), \n",
    "        \"choices_split_words_embeddings_tensor\": choices_split_words_embeddings_tensor, \n",
    "        \"choices_split_words_ids_tensor\": choices_split_words_ids_tensor, \n",
    "        \"choices_outputs\": choices_outputs}\n",
    "      loss = None\n",
    "      train_op = None\n",
    "      eval_metric_ops = None\n",
    "\n",
    "      # Create export outputs\n",
    "      export_outputs = {\n",
    "        \"predict_export_outputs\": tf.estimator.export.PredictOutput(\n",
    "          outputs = predictions_dict)}\n",
    "\n",
    "  # Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "  mode = mode,\n",
    "  predictions = predictions_dict,\n",
    "  loss = loss,\n",
    "  train_op = train_op,\n",
    "  eval_metric_ops = eval_metric_ops,\n",
    "  export_outputs = export_outputs,\n",
    "  scaffold = tf.train.Scaffold(\n",
    "    init_feed_dict = {\n",
    "      embeddings_placeholder: pre_trained_embeddings_array}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create our serving input function to accept the data at serving and send it in the right format to our custom estimator\n",
    "def serving_input_fn():\n",
    "  # Create placeholders to accept the data sent to the model at serving time\n",
    "  feature_placeholders = {\n",
    "    # All features come in as a batch of strings, shape = (batch_size,)\n",
    "    # This was so because of passing the arrays to online ml-engine prediction\n",
    "    feature: tf.placeholder(\n",
    "      dtype = tf.string, \n",
    "      shape = [None]) \n",
    "    for feature in CSV_COLUMNS[0:-1]\n",
    "  }\n",
    "\n",
    "  # Create feature tensors\n",
    "  features = feature_placeholders\n",
    "\n",
    "  return tf.estimator.export.ServingInputReceiver(\n",
    "    features = features, \n",
    "    receiver_tensors = feature_placeholders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create estimator to train and evaluate\n",
    "def train_and_evaluate(args):\n",
    "  # Modify run config\n",
    "  run_config = tf.estimator.RunConfig(\n",
    "    save_checkpoints_steps = 100,\n",
    "    keep_checkpoint_max = 10)\n",
    "  \n",
    "  # Create our custom estimator using our model function\n",
    "  estimator = tf.estimator.Estimator(\n",
    "    model_fn = dual_lstm_text_question_answer,\n",
    "    model_dir = args[\"output_dir\"],\n",
    "    config = run_config,\n",
    "    params = {\n",
    "      \"batch_size\": args[\"batch_size\"],\n",
    "      \"reverse_feature_sequence\": args[\"reverse_feature_sequence\"],\n",
    "      \n",
    "      \"question_lstm_hidden_units\": args[\"question_lstm_hidden_units\"],\n",
    "      \"question_lstm_dropout_output_keep_probs\": args[\"question_lstm_dropout_output_keep_probs\"],\n",
    "      \"choices_lstm_hidden_units\": args[\"choices_lstm_hidden_units\"],\n",
    "      \"choices_lstm_dropout_output_keep_probs\": args[\"choices_lstm_dropout_output_keep_probs\"],\n",
    "\n",
    "      \"use_question_dnn\": args[\"use_question_dnn\"],\n",
    "      \"question_dnn_hidden_units\": args[\"question_dnn_hidden_units\"],\n",
    "      \"question_dnn_dropout_rates\": args[\"question_dnn_dropout_rates\"],\n",
    "      \"use_choices_dnn\": args[\"use_choices_dnn\"],\n",
    "      \"choices_dnn_hidden_units\": args[\"choices_dnn_hidden_units\"],\n",
    "      \"choices_dnn_dropout_rates\": args[\"choices_dnn_dropout_rates\"],\n",
    "      \"final_logits_size\": args[\"final_logits_size\"], \n",
    "\n",
    "      \"learning_rate\": args[\"learning_rate\"]})\n",
    "  \n",
    "  # Create train spec to read in our training data\n",
    "  train_spec = tf.estimator.TrainSpec(\n",
    "    input_fn = read_dataset(\n",
    "      filename = args[\"train_file_pattern\"], \n",
    "      mode = tf.estimator.ModeKeys.TRAIN, \n",
    "      batch_size = args[\"batch_size\"],\n",
    "      params = args),\n",
    "    max_steps = args[\"train_steps\"])\n",
    "  \n",
    "  # Create exporter to save out the complete model to disk\n",
    "  exporter = tf.estimator.LatestExporter(\n",
    "    name = \"exporter\", \n",
    "    serving_input_receiver_fn = serving_input_fn)\n",
    "  \n",
    "  # Create eval spec to read in our validation data and export our model\n",
    "  eval_spec = tf.estimator.EvalSpec(\n",
    "    input_fn = read_dataset(\n",
    "      filename = args[\"eval_file_pattern\"], \n",
    "      mode = tf.estimator.ModeKeys.EVAL, \n",
    "      batch_size = args[\"batch_size\"],\n",
    "      params = args),\n",
    "    steps = None,\n",
    "    start_delay_secs = args[\"start_delay_secs\"], # start evaluating after N seconds\n",
    "    throttle_secs = args[\"throttle_secs\"],  # evaluate every N seconds\n",
    "    exporters = exporter)\n",
    "  \n",
    "  # Create train and evaluate loop to train and evaluate our estimator\n",
    "  tf.estimator.train_and_evaluate(\n",
    "    estimator = estimator, \n",
    "    train_spec = train_spec, \n",
    "    eval_spec = eval_spec)\n",
    "  \n",
    "  return estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 'trained_model', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 10, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x1c935ba978>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 100 or save_checkpoints_secs None.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "train_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "train\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./train_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.shuffle(buffer_size = 10 * batch_size) = \n",
      "<ShuffleDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "train\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c91824358>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb457e7208>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb457e7710>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb457b91d0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb457e7a90>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb457b9f60>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb457b90b8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb457b9630>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb457b9630>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c93a781d0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c93b0c940>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c93b0c9e8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c93a78860>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c93b0c978>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c93b0cba8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c93b0ca90>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c93b0ca90>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into trained_model/model.ckpt.\n",
      "INFO:tensorflow:loss = 7.1548457e-06, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 100 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb46784588>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb46779eb8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c6b7a8080>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c6b7a8278>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb46779be0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c6b7a8630>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c6b7a8710>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c6b7a8518>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c6b7a8518>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb449de860>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c6be1e550>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c6be1eda0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb449ded68>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43f6cac8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4277a278>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43f6ccc0>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43f6ccc0>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:15:10\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-100\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:15:17\n",
      "INFO:tensorflow:Saving dict for global step 100: accuracy = 0.27719298, global_step = 100, loss = 3.4157258e-06, mean_cosine_distance = 2.0, rmse = 0.0018481681\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 100: trained_model/model.ckpt-100\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c52736208>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c527ad6d8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c527ad3c8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c527ad5f8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c52769978>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c527ad7b8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c527ad6a0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c527ada90>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c527ada90>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c93343a90>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c933dac50>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c933daeb8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c93343e80>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c933dae48>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c933daf60>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c933dad30>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c933dad30>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-100\n",
      "WARNING:tensorflow:From /Users/ryangillard/anaconda3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py:1044: calling SavedModelBuilder.add_meta_graph_and_variables (from tensorflow.python.saved_model.builder_impl) with legacy_init_op is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Pass your op to the equivalent parameter main_op instead.\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550729717'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550729717'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.51279\n",
      "INFO:tensorflow:loss = 3.4157242e-06, step = 101 (66.105 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 200 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c950136a0>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c9512b438>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c9512b048>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c9512bc88>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c9512b7b8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c9512be48>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c9512bbe0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c9512bfd0>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c9512bfd0>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c914d0748>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c914d0cf8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c91501780>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c91566eb8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c91501c88>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c91501400>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c91501908>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c91501908>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:16:12\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-200\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:16:20\n",
      "INFO:tensorflow:Saving dict for global step 200: accuracy = 0.27719298, global_step = 200, loss = 1.0511241e-10, mean_cosine_distance = 2.0, rmse = 1.0252435e-05\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 200: trained_model/model.ckpt-200\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb418efa90>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb467316d8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb467313c8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb467315f8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb41dac978>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb467317b8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb46731a90>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb46731550>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb46731550>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94d7aa58>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94de1c18>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94de1e80>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94d7aeb8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94d8e828>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94de1e10>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94de1fd0>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94de1fd0>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-200\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550729780'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550729780'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.60089\n",
      "INFO:tensorflow:loss = 1.0511245e-10, step = 201 (62.465 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 300 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb4479ab00>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43113208>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb431138d0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43113be0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43113898>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb431137f0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43113710>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43113e80>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43113e80>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c95411cf8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c95411c50>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42f0cf98>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43049a58>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42f0ceb8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42f0cc88>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f0cac8>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f0cac8>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:17:13\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-300\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:17:19\n",
      "INFO:tensorflow:Saving dict for global step 300: accuracy = 0.27719298, global_step = 300, loss = 5.073279e-15, mean_cosine_distance = 1.0202931, rmse = 7.122696e-08\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 300: trained_model/model.ckpt-300\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb419cc048>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb46735ef0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb41e770b8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb41e772b0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb46735c18>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb41e77668>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb41e77898>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb41e77550>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb41e77550>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43301898>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb454c4588>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb454c4eb8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43301da0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94d97b00>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4598c2b0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94d97cf8>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94d97cf8>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-300\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550729839'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550729839'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.66892\n",
      "INFO:tensorflow:loss = 5.0732776e-15, step = 301 (59.919 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 400 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c52764e10>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94c6d0f0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94c6d160>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94c6d940>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94c6d470>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94c6db00>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94c6dbe0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94c6d898>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94c6d898>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42dc2f98>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42dc2eb8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42dc29b0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43487940>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c8d6aa048>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c8d6aad68>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c8d6aabe0>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c8d6aabe0>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:18:15\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-400\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:18:22\n",
      "INFO:tensorflow:Saving dict for global step 400: accuracy = 0.27719298, global_step = 400, loss = 1.097652e-19, mean_cosine_distance = 1.0000005, rmse = 3.3130831e-10\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 400: trained_model/model.ckpt-400\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c6bd0d978>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c528bff98>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb41fca0f0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb41fca518>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c528bfc18>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c528bf4a8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb41fca630>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb41fca940>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb41fca940>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb434f5a58>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c946f3c18>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c946f3e80>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb434f5eb8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c6bc5b828>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c946f3e10>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c946f3fd0>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c946f3fd0>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-400\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550729902'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550729902'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.6088\n",
      "INFO:tensorflow:loss = 1.0976519e-19, step = 401 (62.158 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 500 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb44bd2470>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4382a208>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4382a1d0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4382aa58>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4382a588>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4382ac18>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4382acf8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4382a9b0>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4382a9b0>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb427bacf8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb427bac50>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42f11f98>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb428eda58>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42f11eb8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42f11c88>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f11ac8>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f11ac8>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:19:15\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-500\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:19:22\n",
      "INFO:tensorflow:Saving dict for global step 500: accuracy = 0.27719298, global_step = 500, loss = 2.8218201e-24, mean_cosine_distance = 1.0, rmse = 1.6798275e-12\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 500: trained_model/model.ckpt-500\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c6ba793c8>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c6bd73f28>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c9529a0f0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c9529a2e8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c6bd73c50>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c9529a6a0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c9529a860>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c9529a978>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c9529a978>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94d7f9e8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4327dba8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4327de10>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94d7fac8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4327dda0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c6b887e80>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4327dc18>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4327dc18>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-500\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550729962'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550729962'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.66036\n",
      "INFO:tensorflow:loss = 2.8218213e-24, step = 501 (60.228 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 600 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c913b4d68>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c951cf7f0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c9313ab00>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c947057b8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c951cf9e8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c947056a0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94705ac8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c947059e8>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c947059e8>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43579c88>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43579be0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4353ff60>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42d939e8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4353ff28>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4353ffd0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4353ff98>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4353ff98>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:20:16\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-600\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:20:22\n",
      "INFO:tensorflow:Saving dict for global step 600: accuracy = 0.27719298, global_step = 600, loss = 1.2610463e-28, mean_cosine_distance = 1.0, rmse = 1.1229632e-14\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 600: trained_model/model.ckpt-600\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c95282978>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb420a5ef0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb45c2a0b8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb45c2a2b0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb420a5c18>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb45c2a668>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb45c2a898>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb45c2a400>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb45c2a400>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb45c498d0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb434e25c0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb434e2ef0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb45c49dd8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb437f5b38>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb45ba62e8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb437f5d30>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb437f5d30>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-600\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550730022'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550730022'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.65269\n",
      "INFO:tensorflow:loss = 1.2610464e-28, step = 601 (60.508 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 700 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb430b4390>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c92fbc128>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c92fbc048>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c92fbc978>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c92fbc4a8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c92fbcb38>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c92fbc6a0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c92fbccc0>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c92fbccc0>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43233eb8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43233ef0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb432339e8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb435fc978>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4305e080>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4305eda0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4305ec18>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4305ec18>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:21:16\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-700\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:21:22\n",
      "INFO:tensorflow:Saving dict for global step 700: accuracy = 0.27719298, global_step = 700, loss = 2.3181926e-33, mean_cosine_distance = 1.0, rmse = 4.8147613e-17\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 700: trained_model/model.ckpt-700\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c6bbbd048>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c6be3cf28>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb449e0860>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb449e0d30>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c6be3cc50>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb449e0cf8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb449e0780>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb449e0cc0>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb449e0cc0>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb436af4a8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb436af630>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb436af668>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94de58d0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94de5dd8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42fce2e8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c946ffa20>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c946ffa20>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-700\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550730082'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550730082'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.66781\n",
      "INFO:tensorflow:loss = 2.3181928e-33, step = 701 (59.958 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 800 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb4329a4a8>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb438027b8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb438027f0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43802b00>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb438024a8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43802710>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43802a20>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43802da0>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43802da0>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43542eb8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43542ef0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb435429e8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42d95978>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb467da080>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb467dada0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb467dac18>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb467dac18>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:22:18\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-800\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:22:25\n",
      "INFO:tensorflow:Saving dict for global step 800: accuracy = 0.27719298, global_step = 800, loss = 8.6594046e-38, mean_cosine_distance = 1.0, rmse = 2.9426865e-19\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 800: trained_model/model.ckpt-800\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c92b049b0>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb431e0ef0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94ef40b8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94ef42b0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb431e0c18>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94ef4668>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94ef4898>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94ef4400>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94ef4400>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c933f4ac8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c933f4c88>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c933f4ef0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c6b8e66d8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c933f4fd0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c933f4cc0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c933f4f98>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c933f4f98>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-800\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550730145'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550730145'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.60418\n",
      "INFO:tensorflow:loss = 8.6594023e-38, step = 801 (62.337 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 900 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb45969438>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4455a1d0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4455a198>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb4455aa20>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4455a550>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4455abe0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4455aeb8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4455a978>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb4455a978>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb435fdcc0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb435fdc18>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42f2bf28>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4303aa20>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42f2be80>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42f2bc50>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f2ba90>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f2ba90>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:23:20\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-900\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:23:26\n",
      "INFO:tensorflow:Saving dict for global step 900: accuracy = 0.27719298, global_step = 900, loss = 0.0, mean_cosine_distance = 1.0, rmse = 0.0\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 900: trained_model/model.ckpt-900\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c953c6048>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c9525ff98>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94e7b0f0>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94e7b518>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c9525fc18>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c9525f4a8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94e7b630>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94e7b1d0>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94e7b1d0>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43f4f9e8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94f5aba8>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94f5ae10>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43f4fac8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94f5ada0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94e05e80>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94f5ac18>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94f5ac18>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-900\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550730206'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550730206'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.61713\n",
      "INFO:tensorflow:loss = 0.0, step = 901 (61.839 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into trained_model/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: mode = \n",
      "eval\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8d91deb8>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c91505198>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c91505160>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c915059e8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c91505518>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c91505ba8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c91505b00>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c91505940>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c91505940>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb434d2c88>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43441f60>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43441f28>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb434d2da0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43441e80>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43441a58>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43441e48>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb43441e48>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: answer_logits = \n",
      "Tensor(\"map_6/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 16), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-21-06:24:21\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-21-06:24:28\n",
      "INFO:tensorflow:Saving dict for global step 1000: accuracy = 0.27719298, global_step = 1000, loss = 0.0, mean_cosine_distance = 1.0, rmse = 0.0\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1000: trained_model/model.ckpt-1000\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb43a370b8>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb43a2cf60>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94da0048>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c94da0668>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb43a2cc88>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94da0588>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94da0630>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94da05c0>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c94da05c0>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb45752b00>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb45752c88>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb45752ef0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c94f32518>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb45752fd0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb45752cc0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42ff4e80>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42ff4e80>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-1000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model/export/exporter/temp-b'1550730268'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-b'1550730268'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "# Run the model\n",
    "shutil.rmtree(arguments[\"output_dir\"], ignore_errors = True) # start fresh each time\n",
    "estimator = train_and_evaluate(arguments)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "16\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model', 'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'train_steps': 1000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "dual_lstm_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "dual_lstm_text_question_answer: labels = \n",
      "None\n",
      "dual_lstm_text_question_answer: mode = \n",
      "infer\n",
      "dual_lstm_text_question_answer: params = \n",
      "{'batch_size': 16, 'reverse_feature_sequence': True, 'question_lstm_hidden_units': [256, 128, 64], 'question_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'choices_lstm_hidden_units': [256, 128, 64], 'choices_lstm_dropout_output_keep_probs': [0.99, 0.97, 0.95], 'use_question_dnn': True, 'question_dnn_hidden_units': [64, 32, 16], 'question_dnn_dropout_rates': [0.01, 0.03, 0.05], 'use_choices_dnn': True, 'choices_dnn_hidden_units': [64, 32, 16], 'choices_dnn_dropout_rates': [0.01, 0.03, 0.05], 'final_logits_size': 16, 'learning_rate': 0.1}\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "dual_lstm_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "dual_lstm_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "dual_lstm_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c92dc0208>\n",
      "dual_lstm_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "dual_lstm_text_question_answer: current_batch_size32 = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int32)\n",
      "dual_lstm_text_question_answer: current_batch_size64 = \n",
      "Tensor(\"strided_slice_8:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_split_words_embeddings_tensor.shape = \n",
      "(?, ?, ?, 128)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"ReverseSequence:0\", shape=(?, ?, 128), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_split_words_embeddings_tensor.shape = \n",
      "(?, ?, 128)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c92d38710>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c92d38828>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x1c92d38630>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c92d387b8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c92d386a0>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0x1c92d382b0>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c92d383c8>\n",
      "\n",
      "dual_lstm_text_question_answer: question_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0x1c92d383c8>\n",
      "dual_lstm_text_question_answer: question_outputs = \n",
      "Tensor(\"rnn/transpose_1:0\", shape=(?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_states = \n",
      "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 256) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 256) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 128) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 128) dtype=float64>), LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 64) dtype=float64>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 64) dtype=float64>))\n",
      "dual_lstm_text_question_answer: question_outputs.shape = \n",
      "(?, ?, 64)\n",
      "dual_lstm_text_question_answer: question_final_outputs = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_final_outputs.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_1/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_1/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dense_2/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: question_dnn_network = Tensor(\"dropout_2/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: question_logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: question_logits.shape = \n",
      "(?, 16)\n",
      "\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42c73898>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42c4d240>, <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0xb42c4dd68>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: dropout_lstm_cells = \n",
      "[<tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42c739e8>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb42f2cf28>, <tensorflow.python.ops.rnn_cell_impl.DropoutWrapper object at 0xb4260ceb8>]\n",
      "dual_lstm_text_question_answer: create_LSTM_stack: stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f2ca90>\n",
      "\n",
      "dual_lstm_text_question_answer: choices_stacked_lstm_cells = \n",
      "<tensorflow.python.ops.rnn_cell_impl.MultiRNNCell object at 0xb42f2ca90>\n",
      "dual_lstm_text_question_answer: choices_outputs = \n",
      "Tensor(\"while/Exit_5:0\", shape=(?, ?, ?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_batch_indices_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: tiled_choice_indices = \n",
      "Tensor(\"Tile:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_number_of_words_flattened = \n",
      "Tensor(\"Reshape_4:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_final_outputs_flat.shape = \n",
      "(?, 64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = \n",
      "Tensor(\"map_2/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 64), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_4/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_3/Identity:0\", shape=(?, 64), dtype=float64), dropout_rate = 0.01\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_5/Relu:0\", shape=(?, 32), dtype=float64), units = 32\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_4/Identity:0\", shape=(?, 32), dtype=float64), dropout_rate = 0.03\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dense_6/Relu:0\", shape=(?, 16), dtype=float64), units = 16\n",
      "dual_lstm_text_question_answer: choices_dnn_network = Tensor(\"dropout_5/Identity:0\", shape=(?, 16), dtype=float64), dropout_rate = 0.05\n",
      "dual_lstm_text_question_answer: choices_logits_flat = \n",
      "Tensor(\"dense_7/BiasAdd:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_flat.shape = \n",
      "(?, 16)\n",
      "dual_lstm_text_question_answer: choices_logits.shape = \n",
      "(?, ?, 16)\n",
      "dual_lstm_text_question_answer: question_logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: choices_logits_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 16), dtype=float64)\n",
      "dual_lstm_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: euclidean_distances = \n",
      "Tensor(\"transpose_1:0\", shape=(?, ?), dtype=float64)\n",
      "dual_lstm_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"map_5/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?,), dtype=int64)\n",
      "dual_lstm_text_question_answer: predicted_answer_text = \n",
      "Tensor(\"GatherNd:0\", shape=(?,), dtype=string)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "predictions_generator = estimator.predict(input_fn = read_dataset(filename = arguments['eval_file_pattern'], \n",
    "                                                                  mode = tf.estimator.ModeKeys.EVAL, \n",
    "                                                                  batch_size = arguments['batch_size'],\n",
    "                                                                  params = arguments))\n",
    "predictions_list = [x for x in predictions_generator]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cosine_similarities': array([-1.96403813e-34, -1.96403813e-34, -1.96403813e-34, -1.96403813e-34]),\n",
       " 'euclidean_distances': array([2.80288289e-23, 2.80288289e-23, 2.80288289e-23, 2.80288289e-23]),\n",
       " 'predicted_answer_index': 0,\n",
       " 'question_text': b'<s> Which technology was developed most recently </s>',\n",
       " 'predicted_answer_text': b'<s> cellular telephone </s>',\n",
       " 'question_logits': array([ 3.43998133e-25, -3.02262859e-24, -1.78893753e-24, -2.49397701e-24,\n",
       "        -1.73772735e-24,  6.54552521e-24, -1.97741558e-25,  5.74444957e-24,\n",
       "        -4.16338954e-25,  1.53544742e-24, -1.99252766e-24,  2.73598812e-24,\n",
       "         5.54520819e-24,  3.83728911e-24, -5.18451358e-24,  3.53377127e-24]),\n",
       " 'choices_logits': array([[-3.43998133e-25,  3.02262859e-24,  1.78893753e-24,\n",
       "          2.49397701e-24,  1.73772735e-24, -6.54552521e-24,\n",
       "          1.97741558e-25, -5.74444957e-24,  4.16338954e-25,\n",
       "         -1.53544742e-24,  1.99252766e-24, -2.73598812e-24,\n",
       "         -5.54520819e-24, -3.83728911e-24,  5.18451358e-24,\n",
       "         -3.53377127e-24],\n",
       "        [-3.43998133e-25,  3.02262859e-24,  1.78893753e-24,\n",
       "          2.49397701e-24,  1.73772735e-24, -6.54552521e-24,\n",
       "          1.97741558e-25, -5.74444957e-24,  4.16338954e-25,\n",
       "         -1.53544742e-24,  1.99252766e-24, -2.73598812e-24,\n",
       "         -5.54520819e-24, -3.83728911e-24,  5.18451358e-24,\n",
       "         -3.53377127e-24],\n",
       "        [-3.43998133e-25,  3.02262859e-24,  1.78893753e-24,\n",
       "          2.49397701e-24,  1.73772735e-24, -6.54552521e-24,\n",
       "          1.97741558e-25, -5.74444957e-24,  4.16338954e-25,\n",
       "         -1.53544742e-24,  1.99252766e-24, -2.73598812e-24,\n",
       "         -5.54520819e-24, -3.83728911e-24,  5.18451358e-24,\n",
       "         -3.53377127e-24],\n",
       "        [-3.43998133e-25,  3.02262859e-24,  1.78893753e-24,\n",
       "          2.49397701e-24,  1.73772735e-24, -6.54552521e-24,\n",
       "          1.97741558e-25, -5.74444957e-24,  4.16338954e-25,\n",
       "         -1.53544742e-24,  1.99252766e-24, -2.73598812e-24,\n",
       "         -5.54520819e-24, -3.83728911e-24,  5.18451358e-24,\n",
       "         -3.53377127e-24]]),\n",
       " 'choices_number_of_sentences': 4,\n",
       " 'choices_number_of_words': array([4, 3, 3, 3]),\n",
       " \"features['choices']\": b'<s> cellular telephone </s>;<s> television </s>;<s> refrigerator </s>;<s> airplane </s>',\n",
       " 'max_number_of_choices_across_batch': 4,\n",
       " 'max_number_of_choice_words_across_batch': 9,\n",
       " 'choices_split_words_strings_tensor': array([[b'<s>', b'cellular', b'telephone', b'</s>', b'ZYXW_w', b'ZYXW_w',\n",
       "         b'ZYXW_w', b'ZYXW_w', b'ZYXW_w'],\n",
       "        [b'<s>', b'television', b'</s>', b'ZYXW_w', b'ZYXW_w', b'ZYXW_w',\n",
       "         b'ZYXW_w', b'ZYXW_w', b'ZYXW_w'],\n",
       "        [b'<s>', b'refrigerator', b'</s>', b'ZYXW_w', b'ZYXW_w',\n",
       "         b'ZYXW_w', b'ZYXW_w', b'ZYXW_w', b'ZYXW_w'],\n",
       "        [b'<s>', b'airplane', b'</s>', b'ZYXW_w', b'ZYXW_w', b'ZYXW_w',\n",
       "         b'ZYXW_w', b'ZYXW_w', b'ZYXW_w']], dtype=object),\n",
       " 'choices_split_sentences_strings_tensor': array([b'<s> cellular telephone </s>', b'<s> television </s>',\n",
       "        b'<s> refrigerator </s>', b'<s> airplane </s>'], dtype=object),\n",
       " 'choices_final_outputs_flat': array([[-1.29680628e-14, -0.00000000e+00,  3.51115973e-28,\n",
       "          1.70630964e-15, -2.38682894e-19,  5.24808141e-25,\n",
       "          6.31547251e-19, -2.13300200e-26, -3.09041033e-26,\n",
       "          2.36145307e-28, -2.13133277e-26,  3.96785886e-27,\n",
       "         -7.10130358e-18,  9.06840155e-23, -2.96283617e-24,\n",
       "          9.24435356e-31,  2.98586050e-26,  1.05192559e+00,\n",
       "         -1.89343271e-16,  1.05192559e+00, -7.53511844e-19,\n",
       "          3.73266030e-19, -3.59496843e-27, -0.00000000e+00,\n",
       "          1.54580691e-15,  7.86513639e-15,  9.79996397e-17,\n",
       "         -1.05192559e+00, -4.75685304e-26,  1.83766632e-19,\n",
       "          0.00000000e+00,  4.25861075e-11,  2.34894631e-17,\n",
       "          2.12271256e-27,  4.84441375e-19,  1.34347982e-25,\n",
       "         -1.05192559e+00, -2.91057414e-26,  4.93029062e-19,\n",
       "          3.24180437e-17, -1.17230132e-27, -0.00000000e+00,\n",
       "         -2.70101717e-07,  1.36423725e-22, -1.40890067e-24,\n",
       "          4.65831389e-34, -3.73229293e-27, -1.05187493e+00,\n",
       "         -0.00000000e+00, -6.20994924e-26,  3.80183509e-30,\n",
       "          3.46622341e-28,  4.99162337e-19,  0.00000000e+00,\n",
       "         -1.52465060e-29, -2.83200437e-33, -1.06744287e-24,\n",
       "          1.16164766e-24,  8.39736797e-25,  8.04942167e-25,\n",
       "         -8.41128569e-27,  0.00000000e+00,  6.64041468e-15,\n",
       "          5.50516858e-26],\n",
       "        [-1.55717608e-14, -2.83386148e-25,  4.67984953e-28,\n",
       "          2.26911666e-15, -2.94163017e-19,  3.34650243e-25,\n",
       "          8.04355846e-19, -2.62356018e-26, -5.04433526e-26,\n",
       "          3.00316469e-28, -2.71055903e-26,  1.04726987e-26,\n",
       "         -9.94142568e-18,  1.17740461e-22, -2.75296881e-24,\n",
       "          0.00000000e+00,  4.15562057e-26,  1.04742606e+00,\n",
       "         -2.69981633e-16,  1.04742607e+00, -2.75762619e-18,\n",
       "          5.43119798e-19, -7.42711807e-27, -5.19324416e-23,\n",
       "          2.19700550e-15,  1.00145630e-14,  1.16454714e-16,\n",
       "         -1.04742607e+00, -6.11071184e-26,  2.30358515e-19,\n",
       "          2.69718015e-26,  3.14244443e-11,  3.62182667e-17,\n",
       "          2.91622409e-27,  4.89685966e-19,  1.90500593e-25,\n",
       "         -1.04742607e+00, -3.65229430e-26,  5.89627944e-19,\n",
       "          3.60416113e-17, -2.20506745e-27, -2.15962314e-25,\n",
       "         -2.99132729e-07,  0.00000000e+00, -2.17507243e-24,\n",
       "          1.28702882e-33, -5.32305452e-27, -1.04734214e+00,\n",
       "         -2.13012308e-16, -8.59681764e-26,  8.61744869e-30,\n",
       "          1.01500282e-27,  7.26669694e-19,  6.56025577e-31,\n",
       "         -3.10919121e-29, -5.04582052e-33, -1.01137447e-24,\n",
       "          9.75152256e-25,  9.91301391e-25,  1.44051555e-24,\n",
       "         -1.38555665e-26,  8.01563893e-33,  7.07961331e-15,\n",
       "          7.00003003e-26],\n",
       "        [-4.48266691e-14, -6.72784921e-24,  3.14254234e-27,\n",
       "          8.01096927e-15, -1.08573376e-18,  9.01957249e-24,\n",
       "          4.87299336e-18, -4.73201559e-25, -4.64276623e-25,\n",
       "          4.24869071e-27, -2.43139862e-25,  8.16238156e-27,\n",
       "         -1.98014432e-17,  6.81034029e-22, -3.60655091e-23,\n",
       "          1.07539770e-29,  3.93556586e-25,  1.04742606e+00,\n",
       "         -6.69315270e-16,  1.04742607e+00, -5.50138506e-19,\n",
       "          2.53192064e-18, -1.75076437e-26, -1.66935062e-21,\n",
       "          6.15436621e-15,  2.77885073e-14,  3.70741267e-16,\n",
       "         -1.04742607e+00, -5.85118122e-25,  1.08655299e-18,\n",
       "          1.53371157e-25,  1.22049402e-10,  4.84442980e-17,\n",
       "          1.79713970e-26,  4.70371061e-17,  2.03684349e-24,\n",
       "         -1.04742607e+00, -0.00000000e+00,  2.34859307e-18,\n",
       "          1.63584874e-16, -2.15232721e-26, -1.62632479e-24,\n",
       "         -3.70700922e-07,  2.05770905e-21, -6.67576246e-24,\n",
       "          9.64710523e-33, -3.48072059e-26, -1.04737211e+00,\n",
       "         -7.67205797e-16, -8.07844136e-25,  0.00000000e+00,\n",
       "          1.97779200e-27,  3.25429644e-18,  2.07928283e-30,\n",
       "         -1.55818432e-28, -1.38659272e-31, -0.00000000e+00,\n",
       "          1.17470913e-23,  0.00000000e+00,  7.42958652e-25,\n",
       "         -1.04353158e-25,  3.63926039e-32,  3.19727537e-14,\n",
       "          4.95760770e-25],\n",
       "        [-5.70074084e-15, -7.13434392e-26,  4.85104406e-29,\n",
       "          7.46619979e-16, -0.00000000e+00,  8.90450254e-26,\n",
       "          2.21555801e-19, -2.99389449e-27, -0.00000000e+00,\n",
       "          4.33400783e-29, -2.84290468e-27,  1.05337317e-27,\n",
       "         -1.57143420e-18,  1.99466423e-23, -3.99555358e-25,\n",
       "          1.30465376e-31,  4.18804578e-27,  1.04742607e+00,\n",
       "         -5.25243955e-17,  1.04742607e+00, -3.52322397e-19,\n",
       "          1.33076080e-19, -6.21046625e-28, -2.21135393e-23,\n",
       "          7.49512295e-16,  3.19998810e-15,  3.77827720e-17,\n",
       "         -1.04742607e+00, -6.24343189e-27,  5.61047504e-20,\n",
       "          2.05928903e-27,  1.58901084e-11,  5.93596125e-18,\n",
       "          2.79797289e-28,  4.69155138e-19,  2.01893264e-26,\n",
       "         -1.04742607e+00, -3.51729946e-27,  1.58058813e-19,\n",
       "          1.19583851e-17, -1.82168399e-28, -2.04870233e-26,\n",
       "         -1.41423373e-07,  2.27341089e-23, -2.91367044e-25,\n",
       "          9.23780081e-35, -5.70033418e-28, -1.04737534e+00,\n",
       "         -6.60492218e-17, -8.66079829e-27,  6.30529070e-31,\n",
       "          8.00727732e-29,  1.76786180e-19,  4.74151072e-32,\n",
       "         -3.21641499e-30, -4.29822338e-34, -1.28422888e-25,\n",
       "          0.00000000e+00,  1.00606072e-25,  5.83665553e-25,\n",
       "         -1.20387547e-27,  5.21598020e-34,  2.85665430e-15,\n",
       "          7.56799676e-27]]),\n",
       " 'choices_split_words_embeddings_tensor': array([[[ 4.35894093e-01, -6.90240786e-01, -3.63538122e-01,\n",
       "          -3.63021537e-01,  4.55120892e-01, -7.03039450e-01,\n",
       "          -3.78184011e-01,  6.91752539e-01,  6.58387589e-01,\n",
       "          -6.48321157e-01,  5.11399275e-01,  5.12656472e-01,\n",
       "           4.80613469e-01, -1.64105073e-01,  3.65695443e-01,\n",
       "           8.04242965e-01, -7.63225795e-01, -6.19414331e-01,\n",
       "           4.66169991e-01,  4.06682278e-01, -6.80576933e-01,\n",
       "          -6.80893170e-01,  3.59521290e-01,  5.33059067e-01,\n",
       "          -7.64523129e-01, -6.19048915e-01,  4.58433162e-01,\n",
       "           4.35360837e-01,  5.20083903e-01, -4.75653245e-01,\n",
       "           3.64381483e-01,  5.52573556e-01,  4.49191976e-01,\n",
       "           6.46801574e-01, -4.96140844e-01, -5.19753217e-01,\n",
       "          -5.58819622e-01,  6.66508600e-01,  4.42678302e-01,\n",
       "          -3.86778111e-01,  4.56099813e-01, -5.09137813e-01,\n",
       "           3.38084237e-01, -4.52465269e-01,  2.78260774e-01,\n",
       "           3.44197138e-01,  4.55803236e-01,  1.99755386e-02,\n",
       "          -4.91386234e-01,  8.98831030e-02, -1.04100208e-01,\n",
       "           4.49400460e-01,  4.82085182e-01,  4.10652393e-01,\n",
       "          -5.98579359e-01, -3.30592220e-01,  3.59946799e-01,\n",
       "          -6.88307632e-01,  5.74124427e-01,  5.77493074e-01,\n",
       "          -3.78693582e-01,  5.59766131e-01,  5.72329358e-01,\n",
       "          -3.76889104e-01, -6.95305196e-01,  5.39324888e-01,\n",
       "          -4.01818998e-01, -7.19655725e-01,  3.85566079e-01,\n",
       "          -5.40136864e-01,  3.46359914e-01, -6.75667835e-02,\n",
       "          -4.36922443e-01,  3.35584348e-01,  6.45012928e-01,\n",
       "          -4.14008943e-01,  5.99850571e-01,  7.47322095e-01,\n",
       "           7.02731327e-01,  4.06153195e-01, -6.01768385e-01,\n",
       "           5.32327758e-01,  3.57198657e-01, -3.93974940e-01,\n",
       "           6.17832096e-01,  1.84628620e-01, -5.64145554e-01,\n",
       "          -3.18788294e-01,  5.85048329e-01,  6.10484615e-01,\n",
       "          -4.68221583e-01,  6.21364272e-01,  4.11142876e-01,\n",
       "           6.06086237e-01,  4.99116924e-01,  4.51191515e-01,\n",
       "          -2.20120260e-01, -4.41964612e-01,  4.59847433e-01,\n",
       "           7.72433737e-01, -3.97975850e-01, -7.20931225e-01,\n",
       "          -4.45347598e-01,  4.77222135e-01,  5.09159065e-01,\n",
       "          -3.36344884e-01, -4.11218478e-01, -5.85669288e-01,\n",
       "           5.21453520e-01, -6.70070085e-01,  6.06808303e-01,\n",
       "           7.89335977e-02,  5.21091432e-01, -6.90851229e-01,\n",
       "           7.16541384e-01,  7.19964795e-01, -3.70475288e-01,\n",
       "          -6.78148456e-01, -1.01601855e-02, -6.19623875e-01,\n",
       "          -6.36625268e-01,  4.64929545e-01,  4.88569434e-01,\n",
       "           5.27178359e-01,  4.19837207e-01, -6.89444085e-01,\n",
       "           7.31133861e-01,  4.88000075e-01],\n",
       "         [-3.11222021e-02, -1.11067565e-02,  1.40216649e-01,\n",
       "           2.44861059e-02, -4.76102829e-02,  4.69466671e-02,\n",
       "           7.70323128e-02,  8.47792253e-02, -1.10755228e-02,\n",
       "           1.77380383e-01, -7.68707693e-02, -7.85206929e-02,\n",
       "          -4.22359668e-02, -1.21113524e-01, -1.13486931e-01,\n",
       "          -5.88773340e-02,  3.39571722e-02, -3.19090509e-03,\n",
       "           1.12894073e-01, -1.21026196e-01, -9.74387974e-02,\n",
       "           1.03387171e-02,  1.07258528e-01,  1.56129032e-01,\n",
       "           5.25439717e-02, -1.26634210e-01, -1.12718508e-01,\n",
       "          -2.89543774e-02, -3.43360985e-03,  6.90774545e-02,\n",
       "           1.86649722e-03, -8.97226334e-02, -1.03834249e-01,\n",
       "          -1.39833793e-01, -1.40687332e-01,  3.31442878e-02,\n",
       "           1.36993483e-01, -1.28905848e-01, -1.13495011e-02,\n",
       "           1.08825706e-01, -1.68544892e-02,  1.77746251e-01,\n",
       "          -4.42234986e-02,  1.57036632e-01, -7.36185834e-02,\n",
       "          -2.89200284e-02, -1.25648291e-03, -8.45460892e-02,\n",
       "           7.94790313e-02,  1.08109944e-01,  3.40454318e-02,\n",
       "          -9.12908539e-02, -6.99975491e-02, -1.14167646e-01,\n",
       "           1.18125249e-02,  9.13042575e-02,  1.01599827e-01,\n",
       "           8.78472030e-02,  1.48089528e-01,  9.90757197e-02,\n",
       "          -1.68618448e-02, -6.17192537e-02, -6.58189654e-02,\n",
       "          -4.01070863e-02, -1.35776952e-01,  6.45337775e-02,\n",
       "           2.52612736e-02,  2.02325359e-02, -2.69687697e-02,\n",
       "          -8.40274617e-02,  2.00486742e-02, -5.92099363e-03,\n",
       "          -1.21832691e-01, -4.16052714e-02,  1.44769043e-01,\n",
       "          -4.10153121e-02,  4.93564196e-02, -4.45370702e-03,\n",
       "           1.47338346e-01, -1.53690698e-02, -6.70600757e-02,\n",
       "          -1.50841832e-01, -2.68861428e-02,  1.38440698e-01,\n",
       "           1.22329310e-01,  1.08823799e-01,  3.29589024e-02,\n",
       "           4.19871584e-02, -7.83656985e-02, -4.82498333e-02,\n",
       "           1.02265909e-01,  1.13077514e-01, -2.08525490e-02,\n",
       "           7.08322301e-02,  2.97937747e-02, -4.74283397e-02,\n",
       "           1.28857404e-01,  1.05425827e-01, -3.85524631e-02,\n",
       "          -9.55311060e-02, -1.26859983e-02, -3.12803872e-02,\n",
       "          -1.34949788e-01, -1.60607442e-01, -8.79641920e-02,\n",
       "           6.35046661e-02,  1.55443130e-02,  1.18380494e-01,\n",
       "          -8.47047716e-02, -9.66330916e-02, -9.58683640e-02,\n",
       "           1.04212798e-02, -7.53225386e-02, -2.08680201e-02,\n",
       "          -7.74632692e-02, -1.08470798e-01,  7.07964823e-02,\n",
       "          -1.16411656e-01,  5.39151356e-02, -1.48701385e-01,\n",
       "          -1.06886305e-01,  8.72442201e-02,  8.74090418e-02,\n",
       "           2.19611675e-02, -1.15907677e-01,  9.74289551e-02,\n",
       "          -9.57022011e-02,  8.10322911e-02],\n",
       "         [-1.69883277e-02, -7.45195076e-02,  4.94657569e-02,\n",
       "           1.16365753e-01, -2.53526848e-02,  1.43866703e-01,\n",
       "          -4.90480699e-02,  8.51524696e-02,  6.09764978e-02,\n",
       "           1.44107416e-02, -1.35339245e-01, -1.17315806e-01,\n",
       "           1.39062345e-01,  3.75862531e-02,  9.17433202e-03,\n",
       "          -3.05207614e-02, -6.26876950e-02,  7.65945911e-02,\n",
       "          -1.43423155e-02,  7.73601234e-03,  1.83760207e-02,\n",
       "          -6.16202839e-02,  1.14698425e-01, -9.51622277e-02,\n",
       "           1.90913677e-02, -4.42734398e-02, -1.50150117e-02,\n",
       "          -1.64191961e-01, -5.57184294e-02,  4.40388732e-02,\n",
       "          -1.75281823e-01,  2.33845450e-02, -1.98169425e-02,\n",
       "          -1.82358101e-02,  5.81374131e-02,  3.33192423e-02,\n",
       "           1.14030004e-01, -1.43356517e-01, -1.45665839e-01,\n",
       "          -1.19007297e-01,  5.89448959e-02,  1.35608027e-02,\n",
       "           8.97291750e-02,  1.20004706e-01, -5.38392588e-02,\n",
       "           9.90613028e-02, -1.14750274e-01, -7.83358663e-02,\n",
       "          -1.13705568e-01,  7.19190165e-02, -3.61451283e-02,\n",
       "          -1.05739072e-01, -2.07815930e-01, -1.13700211e-01,\n",
       "          -1.30385607e-01, -8.74824170e-03, -5.45388795e-02,\n",
       "           1.79588969e-03,  1.22690119e-01, -7.92122483e-02,\n",
       "           5.35131805e-02, -7.06865638e-02,  8.75621587e-02,\n",
       "          -8.54034498e-02, -4.51708026e-02, -6.04116060e-02,\n",
       "           1.43322334e-01, -2.67418064e-02,  4.79930229e-02,\n",
       "           7.20273927e-02,  1.06320836e-01, -4.01637405e-02,\n",
       "           6.06392249e-02,  1.35734007e-01,  4.65111732e-02,\n",
       "           1.70120433e-01,  4.15219776e-02,  1.25773072e-01,\n",
       "           1.42803758e-01,  7.58803822e-03, -1.03022113e-01,\n",
       "           3.75949666e-02, -1.20917588e-01,  1.13165863e-01,\n",
       "          -1.38755977e-01,  1.03054911e-01, -5.35693467e-02,\n",
       "           9.58248377e-02, -4.66886349e-02,  3.69642787e-02,\n",
       "           2.99393497e-02,  1.74681336e-01,  8.96953493e-02,\n",
       "          -3.22536007e-02,  8.29074904e-02, -3.07964440e-02,\n",
       "           4.34929207e-02,  1.22434713e-01,  1.93099193e-02,\n",
       "           1.34972423e-01,  1.47400528e-01, -6.74171373e-02,\n",
       "           8.12530890e-02, -1.87112838e-02,  3.51336189e-02,\n",
       "           1.54159322e-01,  1.86805502e-02,  2.34162211e-02,\n",
       "          -1.65258124e-01,  6.79316074e-02,  1.02390341e-01,\n",
       "          -1.29128546e-01, -5.75839691e-02,  7.43053854e-02,\n",
       "           1.00770526e-01,  2.67988425e-02,  4.70700972e-02,\n",
       "           1.07195839e-01,  1.89228542e-02, -1.12173155e-01,\n",
       "          -3.38275917e-02, -6.06650067e-03,  4.40590978e-02,\n",
       "          -1.10659888e-02,  1.05239138e-01, -4.93541285e-02,\n",
       "           1.17244728e-01, -4.06420380e-02],\n",
       "         [-5.77516675e-01, -6.18089911e-01, -3.52066928e-01,\n",
       "          -4.75861800e-01,  4.96138097e-01,  4.59995451e-01,\n",
       "          -3.77813000e-01,  6.00455976e-01, -4.80238913e-01,\n",
       "          -6.69563201e-01, -4.74795816e-01,  4.62968927e-01,\n",
       "           4.11359972e-01,  4.43111042e-01,  3.59881510e-01,\n",
       "          -6.06320205e-01, -4.21140573e-01, -5.73674742e-01,\n",
       "           5.91024089e-01,  5.37596511e-01, -6.68298457e-01,\n",
       "           6.65245271e-01,  5.96190947e-01,  6.64344289e-01,\n",
       "           3.21412028e-01,  7.42421755e-01, -5.97499860e-01,\n",
       "           5.58377606e-01,  4.81321411e-01, -4.88104737e-01,\n",
       "           3.84848887e-01, -3.43747423e-01,  5.29766784e-01,\n",
       "          -5.43439470e-01, -5.25546634e-01, -3.09674530e-01,\n",
       "          -5.95669751e-01,  6.09231764e-01,  3.93370234e-01,\n",
       "           2.25453324e-01,  5.05395535e-01, -5.27451309e-01,\n",
       "           6.12555528e-01, -3.94444614e-01, -6.74554222e-01,\n",
       "           4.64775408e-01, -9.48248907e-02, -7.68106997e-01,\n",
       "          -5.72268437e-01,  4.92430475e-01, -6.50284640e-01,\n",
       "           5.57865407e-01,  4.93645664e-01,  5.20631009e-01,\n",
       "          -7.10935772e-01, -5.24166934e-01, -5.63016719e-01,\n",
       "           6.14007475e-01, -3.18349779e-01,  5.54783535e-01,\n",
       "           4.85022681e-01,  5.37467909e-01,  5.59441815e-01,\n",
       "          -5.04301906e-01, -5.37377762e-01,  5.47708543e-01,\n",
       "          -4.62781010e-01, -7.17099970e-01,  6.29946812e-01,\n",
       "          -4.78963461e-01, -5.40704428e-01, -3.66731808e-01,\n",
       "          -3.80452086e-01, -4.41724013e-01,  8.18987530e-01,\n",
       "          -4.64774673e-01, -5.79889722e-01,  6.69544320e-01,\n",
       "           2.45410488e-01,  4.52681415e-01, -5.00647311e-01,\n",
       "           3.20979641e-01,  4.61106458e-01, -4.27636043e-01,\n",
       "          -4.40790442e-01, -6.64471064e-01, -5.39096584e-01,\n",
       "          -4.64493383e-01, -4.75951124e-01,  5.14621023e-01,\n",
       "          -4.94421869e-01, -3.61153487e-01,  5.74876242e-01,\n",
       "           5.66119378e-01, -5.97063035e-01,  5.50503647e-01,\n",
       "           5.32057408e-01,  4.12173358e-01,  6.24858655e-01,\n",
       "           5.78696335e-01,  5.53475545e-01,  5.13225235e-01,\n",
       "           5.54929351e-01,  2.85364488e-01,  4.48318645e-01,\n",
       "          -4.59243853e-01, -5.38397295e-01, -5.90250714e-01,\n",
       "           4.54159756e-01,  4.05212476e-01,  7.05584590e-01,\n",
       "           5.25710743e-01,  3.92139717e-01, -5.64833457e-01,\n",
       "          -3.24598260e-01,  4.57404105e-01, -4.45824263e-01,\n",
       "          -5.76573958e-01, -5.55097708e-01, -7.43564099e-01,\n",
       "           4.16153088e-01,  5.14029055e-01,  5.90597933e-01,\n",
       "           3.45760307e-01, -1.09261320e-01, -5.75365886e-01,\n",
       "          -4.94889133e-01,  5.72729304e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01]],\n",
       " \n",
       "        [[ 4.35894093e-01, -6.90240786e-01, -3.63538122e-01,\n",
       "          -3.63021537e-01,  4.55120892e-01, -7.03039450e-01,\n",
       "          -3.78184011e-01,  6.91752539e-01,  6.58387589e-01,\n",
       "          -6.48321157e-01,  5.11399275e-01,  5.12656472e-01,\n",
       "           4.80613469e-01, -1.64105073e-01,  3.65695443e-01,\n",
       "           8.04242965e-01, -7.63225795e-01, -6.19414331e-01,\n",
       "           4.66169991e-01,  4.06682278e-01, -6.80576933e-01,\n",
       "          -6.80893170e-01,  3.59521290e-01,  5.33059067e-01,\n",
       "          -7.64523129e-01, -6.19048915e-01,  4.58433162e-01,\n",
       "           4.35360837e-01,  5.20083903e-01, -4.75653245e-01,\n",
       "           3.64381483e-01,  5.52573556e-01,  4.49191976e-01,\n",
       "           6.46801574e-01, -4.96140844e-01, -5.19753217e-01,\n",
       "          -5.58819622e-01,  6.66508600e-01,  4.42678302e-01,\n",
       "          -3.86778111e-01,  4.56099813e-01, -5.09137813e-01,\n",
       "           3.38084237e-01, -4.52465269e-01,  2.78260774e-01,\n",
       "           3.44197138e-01,  4.55803236e-01,  1.99755386e-02,\n",
       "          -4.91386234e-01,  8.98831030e-02, -1.04100208e-01,\n",
       "           4.49400460e-01,  4.82085182e-01,  4.10652393e-01,\n",
       "          -5.98579359e-01, -3.30592220e-01,  3.59946799e-01,\n",
       "          -6.88307632e-01,  5.74124427e-01,  5.77493074e-01,\n",
       "          -3.78693582e-01,  5.59766131e-01,  5.72329358e-01,\n",
       "          -3.76889104e-01, -6.95305196e-01,  5.39324888e-01,\n",
       "          -4.01818998e-01, -7.19655725e-01,  3.85566079e-01,\n",
       "          -5.40136864e-01,  3.46359914e-01, -6.75667835e-02,\n",
       "          -4.36922443e-01,  3.35584348e-01,  6.45012928e-01,\n",
       "          -4.14008943e-01,  5.99850571e-01,  7.47322095e-01,\n",
       "           7.02731327e-01,  4.06153195e-01, -6.01768385e-01,\n",
       "           5.32327758e-01,  3.57198657e-01, -3.93974940e-01,\n",
       "           6.17832096e-01,  1.84628620e-01, -5.64145554e-01,\n",
       "          -3.18788294e-01,  5.85048329e-01,  6.10484615e-01,\n",
       "          -4.68221583e-01,  6.21364272e-01,  4.11142876e-01,\n",
       "           6.06086237e-01,  4.99116924e-01,  4.51191515e-01,\n",
       "          -2.20120260e-01, -4.41964612e-01,  4.59847433e-01,\n",
       "           7.72433737e-01, -3.97975850e-01, -7.20931225e-01,\n",
       "          -4.45347598e-01,  4.77222135e-01,  5.09159065e-01,\n",
       "          -3.36344884e-01, -4.11218478e-01, -5.85669288e-01,\n",
       "           5.21453520e-01, -6.70070085e-01,  6.06808303e-01,\n",
       "           7.89335977e-02,  5.21091432e-01, -6.90851229e-01,\n",
       "           7.16541384e-01,  7.19964795e-01, -3.70475288e-01,\n",
       "          -6.78148456e-01, -1.01601855e-02, -6.19623875e-01,\n",
       "          -6.36625268e-01,  4.64929545e-01,  4.88569434e-01,\n",
       "           5.27178359e-01,  4.19837207e-01, -6.89444085e-01,\n",
       "           7.31133861e-01,  4.88000075e-01],\n",
       "         [-1.10445820e-01, -1.65292114e-01,  1.54537126e-01,\n",
       "           1.06466241e-01,  5.47727011e-02, -3.47442813e-02,\n",
       "           1.12412199e-01,  1.54294491e-01, -1.98901584e-03,\n",
       "           5.75552471e-02, -1.16661392e-01,  6.18942715e-02,\n",
       "           5.55484183e-02, -1.46791592e-01, -1.89785305e-02,\n",
       "          -4.92822099e-03,  1.05348751e-02,  1.01921670e-01,\n",
       "           9.84728634e-02,  6.89881369e-02, -1.12547604e-02,\n",
       "          -7.90804476e-02, -1.00235447e-01, -1.01331078e-01,\n",
       "           7.58657530e-02, -1.25332296e-01,  4.49509285e-02,\n",
       "           5.92790497e-03,  1.58699706e-01,  1.00063838e-01,\n",
       "          -1.67922080e-01,  9.20813903e-02,  7.06066936e-02,\n",
       "           1.16752721e-01, -3.36600095e-02,  4.71878238e-03,\n",
       "           1.38304025e-01, -1.03947505e-01, -1.42523095e-01,\n",
       "          -5.13488129e-02, -9.18711200e-02,  6.11453652e-02,\n",
       "          -4.66456078e-02,  1.92581981e-01, -1.71438083e-01,\n",
       "          -1.39601395e-01,  1.16933957e-02,  5.09522110e-02,\n",
       "          -1.23350702e-01,  7.15119466e-02, -8.83014798e-02,\n",
       "           1.58163123e-02, -1.11886794e-02,  1.24260113e-01,\n",
       "          -5.39466925e-02, -2.84638908e-02, -3.83044370e-02,\n",
       "          -5.90689890e-02,  1.19153053e-01,  1.32882521e-01,\n",
       "          -8.05074908e-03, -1.03894509e-01,  6.60996744e-03,\n",
       "          -1.38808861e-01, -1.26504585e-01, -8.36068466e-02,\n",
       "           5.31134121e-02, -1.22754171e-01,  5.25002554e-02,\n",
       "          -5.97764887e-02, -3.68335135e-02, -9.28631499e-02,\n",
       "           5.73290065e-02, -3.77744576e-03, -2.42926329e-02,\n",
       "           5.16109653e-02, -1.57160968e-01, -3.32409851e-02,\n",
       "          -8.31977278e-03, -1.04308270e-01, -3.10636219e-02,\n",
       "          -5.43080382e-02, -3.88205014e-02, -8.71001482e-02,\n",
       "           8.04040506e-02,  6.60049096e-02,  1.08306922e-01,\n",
       "          -3.07027157e-02, -1.01434857e-01,  1.71902508e-03,\n",
       "           6.48585185e-02, -2.53918152e-02,  1.17297478e-01,\n",
       "           1.14870116e-01,  8.81925002e-02, -7.57342875e-02,\n",
       "           1.72135979e-01, -4.50807773e-02,  6.71765134e-02,\n",
       "           1.16231889e-01,  8.80740583e-02, -1.47686645e-01,\n",
       "           1.06215909e-01, -8.22030380e-03,  2.29201745e-03,\n",
       "           2.38293223e-02,  2.39331299e-03,  1.06897488e-01,\n",
       "          -2.80531105e-02, -2.57373210e-02, -4.61268239e-02,\n",
       "          -4.59426306e-02, -5.85219525e-02, -3.18328617e-03,\n",
       "           7.36349821e-02, -9.68040526e-02,  2.05736905e-02,\n",
       "           5.74278086e-02,  1.29157364e-01,  1.17925100e-01,\n",
       "          -6.78232312e-02,  8.07757303e-02, -4.21765335e-02,\n",
       "           2.85983849e-02, -1.19884171e-01,  7.55053908e-02,\n",
       "          -1.17295660e-01,  2.08773613e-02],\n",
       "         [-5.77516675e-01, -6.18089911e-01, -3.52066928e-01,\n",
       "          -4.75861800e-01,  4.96138097e-01,  4.59995451e-01,\n",
       "          -3.77813000e-01,  6.00455976e-01, -4.80238913e-01,\n",
       "          -6.69563201e-01, -4.74795816e-01,  4.62968927e-01,\n",
       "           4.11359972e-01,  4.43111042e-01,  3.59881510e-01,\n",
       "          -6.06320205e-01, -4.21140573e-01, -5.73674742e-01,\n",
       "           5.91024089e-01,  5.37596511e-01, -6.68298457e-01,\n",
       "           6.65245271e-01,  5.96190947e-01,  6.64344289e-01,\n",
       "           3.21412028e-01,  7.42421755e-01, -5.97499860e-01,\n",
       "           5.58377606e-01,  4.81321411e-01, -4.88104737e-01,\n",
       "           3.84848887e-01, -3.43747423e-01,  5.29766784e-01,\n",
       "          -5.43439470e-01, -5.25546634e-01, -3.09674530e-01,\n",
       "          -5.95669751e-01,  6.09231764e-01,  3.93370234e-01,\n",
       "           2.25453324e-01,  5.05395535e-01, -5.27451309e-01,\n",
       "           6.12555528e-01, -3.94444614e-01, -6.74554222e-01,\n",
       "           4.64775408e-01, -9.48248907e-02, -7.68106997e-01,\n",
       "          -5.72268437e-01,  4.92430475e-01, -6.50284640e-01,\n",
       "           5.57865407e-01,  4.93645664e-01,  5.20631009e-01,\n",
       "          -7.10935772e-01, -5.24166934e-01, -5.63016719e-01,\n",
       "           6.14007475e-01, -3.18349779e-01,  5.54783535e-01,\n",
       "           4.85022681e-01,  5.37467909e-01,  5.59441815e-01,\n",
       "          -5.04301906e-01, -5.37377762e-01,  5.47708543e-01,\n",
       "          -4.62781010e-01, -7.17099970e-01,  6.29946812e-01,\n",
       "          -4.78963461e-01, -5.40704428e-01, -3.66731808e-01,\n",
       "          -3.80452086e-01, -4.41724013e-01,  8.18987530e-01,\n",
       "          -4.64774673e-01, -5.79889722e-01,  6.69544320e-01,\n",
       "           2.45410488e-01,  4.52681415e-01, -5.00647311e-01,\n",
       "           3.20979641e-01,  4.61106458e-01, -4.27636043e-01,\n",
       "          -4.40790442e-01, -6.64471064e-01, -5.39096584e-01,\n",
       "          -4.64493383e-01, -4.75951124e-01,  5.14621023e-01,\n",
       "          -4.94421869e-01, -3.61153487e-01,  5.74876242e-01,\n",
       "           5.66119378e-01, -5.97063035e-01,  5.50503647e-01,\n",
       "           5.32057408e-01,  4.12173358e-01,  6.24858655e-01,\n",
       "           5.78696335e-01,  5.53475545e-01,  5.13225235e-01,\n",
       "           5.54929351e-01,  2.85364488e-01,  4.48318645e-01,\n",
       "          -4.59243853e-01, -5.38397295e-01, -5.90250714e-01,\n",
       "           4.54159756e-01,  4.05212476e-01,  7.05584590e-01,\n",
       "           5.25710743e-01,  3.92139717e-01, -5.64833457e-01,\n",
       "          -3.24598260e-01,  4.57404105e-01, -4.45824263e-01,\n",
       "          -5.76573958e-01, -5.55097708e-01, -7.43564099e-01,\n",
       "           4.16153088e-01,  5.14029055e-01,  5.90597933e-01,\n",
       "           3.45760307e-01, -1.09261320e-01, -5.75365886e-01,\n",
       "          -4.94889133e-01,  5.72729304e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01]],\n",
       " \n",
       "        [[ 4.35894093e-01, -6.90240786e-01, -3.63538122e-01,\n",
       "          -3.63021537e-01,  4.55120892e-01, -7.03039450e-01,\n",
       "          -3.78184011e-01,  6.91752539e-01,  6.58387589e-01,\n",
       "          -6.48321157e-01,  5.11399275e-01,  5.12656472e-01,\n",
       "           4.80613469e-01, -1.64105073e-01,  3.65695443e-01,\n",
       "           8.04242965e-01, -7.63225795e-01, -6.19414331e-01,\n",
       "           4.66169991e-01,  4.06682278e-01, -6.80576933e-01,\n",
       "          -6.80893170e-01,  3.59521290e-01,  5.33059067e-01,\n",
       "          -7.64523129e-01, -6.19048915e-01,  4.58433162e-01,\n",
       "           4.35360837e-01,  5.20083903e-01, -4.75653245e-01,\n",
       "           3.64381483e-01,  5.52573556e-01,  4.49191976e-01,\n",
       "           6.46801574e-01, -4.96140844e-01, -5.19753217e-01,\n",
       "          -5.58819622e-01,  6.66508600e-01,  4.42678302e-01,\n",
       "          -3.86778111e-01,  4.56099813e-01, -5.09137813e-01,\n",
       "           3.38084237e-01, -4.52465269e-01,  2.78260774e-01,\n",
       "           3.44197138e-01,  4.55803236e-01,  1.99755386e-02,\n",
       "          -4.91386234e-01,  8.98831030e-02, -1.04100208e-01,\n",
       "           4.49400460e-01,  4.82085182e-01,  4.10652393e-01,\n",
       "          -5.98579359e-01, -3.30592220e-01,  3.59946799e-01,\n",
       "          -6.88307632e-01,  5.74124427e-01,  5.77493074e-01,\n",
       "          -3.78693582e-01,  5.59766131e-01,  5.72329358e-01,\n",
       "          -3.76889104e-01, -6.95305196e-01,  5.39324888e-01,\n",
       "          -4.01818998e-01, -7.19655725e-01,  3.85566079e-01,\n",
       "          -5.40136864e-01,  3.46359914e-01, -6.75667835e-02,\n",
       "          -4.36922443e-01,  3.35584348e-01,  6.45012928e-01,\n",
       "          -4.14008943e-01,  5.99850571e-01,  7.47322095e-01,\n",
       "           7.02731327e-01,  4.06153195e-01, -6.01768385e-01,\n",
       "           5.32327758e-01,  3.57198657e-01, -3.93974940e-01,\n",
       "           6.17832096e-01,  1.84628620e-01, -5.64145554e-01,\n",
       "          -3.18788294e-01,  5.85048329e-01,  6.10484615e-01,\n",
       "          -4.68221583e-01,  6.21364272e-01,  4.11142876e-01,\n",
       "           6.06086237e-01,  4.99116924e-01,  4.51191515e-01,\n",
       "          -2.20120260e-01, -4.41964612e-01,  4.59847433e-01,\n",
       "           7.72433737e-01, -3.97975850e-01, -7.20931225e-01,\n",
       "          -4.45347598e-01,  4.77222135e-01,  5.09159065e-01,\n",
       "          -3.36344884e-01, -4.11218478e-01, -5.85669288e-01,\n",
       "           5.21453520e-01, -6.70070085e-01,  6.06808303e-01,\n",
       "           7.89335977e-02,  5.21091432e-01, -6.90851229e-01,\n",
       "           7.16541384e-01,  7.19964795e-01, -3.70475288e-01,\n",
       "          -6.78148456e-01, -1.01601855e-02, -6.19623875e-01,\n",
       "          -6.36625268e-01,  4.64929545e-01,  4.88569434e-01,\n",
       "           5.27178359e-01,  4.19837207e-01, -6.89444085e-01,\n",
       "           7.31133861e-01,  4.88000075e-01],\n",
       "         [ 4.65694927e-02,  3.64306383e-04, -4.55610529e-02,\n",
       "           1.42888546e-01,  1.35810673e-01, -6.19769245e-02,\n",
       "           1.92483980e-02,  1.04349084e-01,  9.91525576e-02,\n",
       "           1.27092740e-02,  1.87204164e-02, -1.32493928e-01,\n",
       "           1.28624052e-01,  6.21092878e-02, -4.96764891e-02,\n",
       "          -2.42200145e-03, -1.97769180e-02,  1.18292077e-02,\n",
       "          -4.37445147e-03, -1.05259903e-01, -1.03435613e-01,\n",
       "          -1.08505890e-01, -1.13816284e-01,  6.58713728e-02,\n",
       "           2.63832603e-02,  1.00985639e-01, -4.61861826e-02,\n",
       "          -1.00525327e-01, -1.11127950e-01,  4.10900079e-02,\n",
       "          -4.38367613e-02, -1.09787256e-01,  9.09944624e-02,\n",
       "          -1.34942442e-01,  7.19672367e-02,  7.76645029e-03,\n",
       "           1.06972031e-01, -1.04207866e-01,  2.82026883e-02,\n",
       "           1.26137123e-01, -1.24997310e-01,  3.20279934e-02,\n",
       "          -6.19260967e-02,  5.34919538e-02, -1.13386884e-01,\n",
       "           4.72239591e-02,  4.65951860e-02, -1.52203187e-01,\n",
       "          -1.13632463e-01, -7.91718587e-02, -4.24016081e-02,\n",
       "          -7.21789300e-02, -1.84695255e-02,  2.34837942e-02,\n",
       "           4.27412875e-02,  1.09068736e-01,  1.52692392e-01,\n",
       "          -9.45710242e-02,  1.64702088e-01, -1.32019268e-02,\n",
       "           9.39843152e-03, -1.06475368e-01,  1.55013040e-01,\n",
       "          -1.75184757e-02, -1.57220781e-01, -1.93745252e-02,\n",
       "           1.35863051e-01,  1.23334132e-01, -4.74065170e-02,\n",
       "          -1.18272923e-01,  1.42185790e-02,  4.34328243e-02,\n",
       "          -4.45812568e-02, -6.00724900e-03, -6.41602576e-02,\n",
       "           3.62064876e-02,  1.56842116e-02,  1.05616972e-02,\n",
       "           1.07749760e-01, -1.89930514e-01,  1.77646317e-02,\n",
       "          -1.16736196e-01, -1.06671229e-01,  6.44992515e-02,\n",
       "           2.60655512e-03, -9.06655267e-02,  1.90089680e-02,\n",
       "          -9.44107547e-02,  1.16575949e-01, -8.90290588e-02,\n",
       "          -9.55928341e-02, -7.49013498e-02,  1.16436593e-01,\n",
       "           2.10826914e-03,  3.51613164e-02, -1.11296490e-01,\n",
       "           3.19585055e-02,  9.50749964e-02,  2.13308781e-02,\n",
       "          -5.11193909e-02,  1.56165734e-01,  2.58519761e-02,\n",
       "           1.00549869e-02, -9.53698382e-02, -2.91100983e-02,\n",
       "           3.38099487e-02,  1.04298189e-01, -3.50684561e-02,\n",
       "           2.05357540e-02, -1.92958534e-01,  1.86512072e-04,\n",
       "          -4.92425673e-02,  6.22710921e-02, -1.35578409e-01,\n",
       "          -3.33761498e-02,  1.08708963e-01,  1.77849501e-01,\n",
       "           9.63941738e-02,  1.34976432e-01,  1.71583891e-02,\n",
       "           6.48749247e-02, -1.08968392e-01,  1.15917057e-01,\n",
       "           6.91401511e-02, -2.31814347e-02,  1.06307618e-01,\n",
       "           1.13178790e-01,  1.40357763e-01],\n",
       "         [-5.77516675e-01, -6.18089911e-01, -3.52066928e-01,\n",
       "          -4.75861800e-01,  4.96138097e-01,  4.59995451e-01,\n",
       "          -3.77813000e-01,  6.00455976e-01, -4.80238913e-01,\n",
       "          -6.69563201e-01, -4.74795816e-01,  4.62968927e-01,\n",
       "           4.11359972e-01,  4.43111042e-01,  3.59881510e-01,\n",
       "          -6.06320205e-01, -4.21140573e-01, -5.73674742e-01,\n",
       "           5.91024089e-01,  5.37596511e-01, -6.68298457e-01,\n",
       "           6.65245271e-01,  5.96190947e-01,  6.64344289e-01,\n",
       "           3.21412028e-01,  7.42421755e-01, -5.97499860e-01,\n",
       "           5.58377606e-01,  4.81321411e-01, -4.88104737e-01,\n",
       "           3.84848887e-01, -3.43747423e-01,  5.29766784e-01,\n",
       "          -5.43439470e-01, -5.25546634e-01, -3.09674530e-01,\n",
       "          -5.95669751e-01,  6.09231764e-01,  3.93370234e-01,\n",
       "           2.25453324e-01,  5.05395535e-01, -5.27451309e-01,\n",
       "           6.12555528e-01, -3.94444614e-01, -6.74554222e-01,\n",
       "           4.64775408e-01, -9.48248907e-02, -7.68106997e-01,\n",
       "          -5.72268437e-01,  4.92430475e-01, -6.50284640e-01,\n",
       "           5.57865407e-01,  4.93645664e-01,  5.20631009e-01,\n",
       "          -7.10935772e-01, -5.24166934e-01, -5.63016719e-01,\n",
       "           6.14007475e-01, -3.18349779e-01,  5.54783535e-01,\n",
       "           4.85022681e-01,  5.37467909e-01,  5.59441815e-01,\n",
       "          -5.04301906e-01, -5.37377762e-01,  5.47708543e-01,\n",
       "          -4.62781010e-01, -7.17099970e-01,  6.29946812e-01,\n",
       "          -4.78963461e-01, -5.40704428e-01, -3.66731808e-01,\n",
       "          -3.80452086e-01, -4.41724013e-01,  8.18987530e-01,\n",
       "          -4.64774673e-01, -5.79889722e-01,  6.69544320e-01,\n",
       "           2.45410488e-01,  4.52681415e-01, -5.00647311e-01,\n",
       "           3.20979641e-01,  4.61106458e-01, -4.27636043e-01,\n",
       "          -4.40790442e-01, -6.64471064e-01, -5.39096584e-01,\n",
       "          -4.64493383e-01, -4.75951124e-01,  5.14621023e-01,\n",
       "          -4.94421869e-01, -3.61153487e-01,  5.74876242e-01,\n",
       "           5.66119378e-01, -5.97063035e-01,  5.50503647e-01,\n",
       "           5.32057408e-01,  4.12173358e-01,  6.24858655e-01,\n",
       "           5.78696335e-01,  5.53475545e-01,  5.13225235e-01,\n",
       "           5.54929351e-01,  2.85364488e-01,  4.48318645e-01,\n",
       "          -4.59243853e-01, -5.38397295e-01, -5.90250714e-01,\n",
       "           4.54159756e-01,  4.05212476e-01,  7.05584590e-01,\n",
       "           5.25710743e-01,  3.92139717e-01, -5.64833457e-01,\n",
       "          -3.24598260e-01,  4.57404105e-01, -4.45824263e-01,\n",
       "          -5.76573958e-01, -5.55097708e-01, -7.43564099e-01,\n",
       "           4.16153088e-01,  5.14029055e-01,  5.90597933e-01,\n",
       "           3.45760307e-01, -1.09261320e-01, -5.75365886e-01,\n",
       "          -4.94889133e-01,  5.72729304e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01]],\n",
       " \n",
       "        [[ 4.35894093e-01, -6.90240786e-01, -3.63538122e-01,\n",
       "          -3.63021537e-01,  4.55120892e-01, -7.03039450e-01,\n",
       "          -3.78184011e-01,  6.91752539e-01,  6.58387589e-01,\n",
       "          -6.48321157e-01,  5.11399275e-01,  5.12656472e-01,\n",
       "           4.80613469e-01, -1.64105073e-01,  3.65695443e-01,\n",
       "           8.04242965e-01, -7.63225795e-01, -6.19414331e-01,\n",
       "           4.66169991e-01,  4.06682278e-01, -6.80576933e-01,\n",
       "          -6.80893170e-01,  3.59521290e-01,  5.33059067e-01,\n",
       "          -7.64523129e-01, -6.19048915e-01,  4.58433162e-01,\n",
       "           4.35360837e-01,  5.20083903e-01, -4.75653245e-01,\n",
       "           3.64381483e-01,  5.52573556e-01,  4.49191976e-01,\n",
       "           6.46801574e-01, -4.96140844e-01, -5.19753217e-01,\n",
       "          -5.58819622e-01,  6.66508600e-01,  4.42678302e-01,\n",
       "          -3.86778111e-01,  4.56099813e-01, -5.09137813e-01,\n",
       "           3.38084237e-01, -4.52465269e-01,  2.78260774e-01,\n",
       "           3.44197138e-01,  4.55803236e-01,  1.99755386e-02,\n",
       "          -4.91386234e-01,  8.98831030e-02, -1.04100208e-01,\n",
       "           4.49400460e-01,  4.82085182e-01,  4.10652393e-01,\n",
       "          -5.98579359e-01, -3.30592220e-01,  3.59946799e-01,\n",
       "          -6.88307632e-01,  5.74124427e-01,  5.77493074e-01,\n",
       "          -3.78693582e-01,  5.59766131e-01,  5.72329358e-01,\n",
       "          -3.76889104e-01, -6.95305196e-01,  5.39324888e-01,\n",
       "          -4.01818998e-01, -7.19655725e-01,  3.85566079e-01,\n",
       "          -5.40136864e-01,  3.46359914e-01, -6.75667835e-02,\n",
       "          -4.36922443e-01,  3.35584348e-01,  6.45012928e-01,\n",
       "          -4.14008943e-01,  5.99850571e-01,  7.47322095e-01,\n",
       "           7.02731327e-01,  4.06153195e-01, -6.01768385e-01,\n",
       "           5.32327758e-01,  3.57198657e-01, -3.93974940e-01,\n",
       "           6.17832096e-01,  1.84628620e-01, -5.64145554e-01,\n",
       "          -3.18788294e-01,  5.85048329e-01,  6.10484615e-01,\n",
       "          -4.68221583e-01,  6.21364272e-01,  4.11142876e-01,\n",
       "           6.06086237e-01,  4.99116924e-01,  4.51191515e-01,\n",
       "          -2.20120260e-01, -4.41964612e-01,  4.59847433e-01,\n",
       "           7.72433737e-01, -3.97975850e-01, -7.20931225e-01,\n",
       "          -4.45347598e-01,  4.77222135e-01,  5.09159065e-01,\n",
       "          -3.36344884e-01, -4.11218478e-01, -5.85669288e-01,\n",
       "           5.21453520e-01, -6.70070085e-01,  6.06808303e-01,\n",
       "           7.89335977e-02,  5.21091432e-01, -6.90851229e-01,\n",
       "           7.16541384e-01,  7.19964795e-01, -3.70475288e-01,\n",
       "          -6.78148456e-01, -1.01601855e-02, -6.19623875e-01,\n",
       "          -6.36625268e-01,  4.64929545e-01,  4.88569434e-01,\n",
       "           5.27178359e-01,  4.19837207e-01, -6.89444085e-01,\n",
       "           7.31133861e-01,  4.88000075e-01],\n",
       "         [ 1.82797089e-02, -1.66848376e-01,  7.37468079e-02,\n",
       "          -5.07283881e-02, -4.42070477e-02, -5.18199727e-02,\n",
       "          -6.89767525e-02,  5.80773540e-02, -2.45996006e-02,\n",
       "           1.51270002e-01,  7.26424605e-02, -1.30241036e-01,\n",
       "           5.22892689e-03, -1.24216259e-01, -1.51618257e-01,\n",
       "          -1.07089914e-01,  9.70846266e-02,  3.89434882e-02,\n",
       "           8.89138319e-03, -1.20072454e-01,  1.27463669e-01,\n",
       "          -1.54876441e-01, -3.43067907e-02,  1.14109039e-01,\n",
       "           2.87331194e-02, -2.97617037e-02, -4.94864322e-02,\n",
       "          -1.65494040e-01,  9.49299410e-02, -9.89464372e-02,\n",
       "          -1.16084874e-01, -1.24984838e-01, -9.32976902e-02,\n",
       "           8.46001506e-02,  2.65642442e-02, -8.66123289e-02,\n",
       "          -6.15692623e-02,  5.07118292e-02,  2.87875813e-02,\n",
       "           6.24859780e-02, -3.31482701e-02,  8.12179223e-02,\n",
       "          -1.03953257e-01,  1.91621594e-02, -1.05245322e-01,\n",
       "           3.06637064e-02,  7.47361332e-02,  7.11633638e-02,\n",
       "           7.92440251e-02,  1.18413396e-01, -1.06755950e-01,\n",
       "          -2.51875576e-02,  2.41174698e-02, -1.04886003e-01,\n",
       "          -4.10967879e-02,  1.62545443e-02, -4.15730663e-02,\n",
       "          -9.34778154e-02,  1.48759156e-01, -3.12085059e-02,\n",
       "          -1.48197234e-01, -3.02801486e-02, -9.44024359e-04,\n",
       "           8.37468058e-02,  1.80729534e-02,  7.00590909e-02,\n",
       "           5.95397539e-02, -1.13457017e-01, -1.06923454e-01,\n",
       "           5.29445820e-02,  3.84344496e-02,  1.74085394e-01,\n",
       "           1.01811551e-01, -6.62217513e-02,  9.14494917e-02,\n",
       "          -7.53805935e-02, -1.37061566e-01,  3.80656086e-02,\n",
       "          -4.28603888e-02, -3.65203805e-02,  8.35944489e-02,\n",
       "          -9.17347446e-02, -1.41115472e-01,  1.43024847e-01,\n",
       "          -2.26930249e-02,  2.92809345e-02, -5.62997162e-02,\n",
       "           7.31841428e-03,  1.67620048e-01,  1.04477428e-01,\n",
       "           2.73121800e-03,  6.01042695e-02, -4.61381190e-02,\n",
       "           1.80693772e-02,  8.61390233e-02, -1.35620758e-01,\n",
       "          -7.51655772e-02, -4.20626290e-02, -5.74698262e-02,\n",
       "           1.53455928e-01,  2.40027178e-02, -8.20656419e-02,\n",
       "           1.79613987e-03, -2.00465322e-01,  1.00492453e-02,\n",
       "           1.49887189e-01, -6.56626076e-02,  1.17403775e-01,\n",
       "          -3.21721695e-02,  2.10841373e-02,  5.99882565e-02,\n",
       "           6.84576780e-02, -8.99834558e-02, -9.13176090e-02,\n",
       "          -4.63911779e-02,  4.58353274e-02,  6.87385574e-02,\n",
       "          -6.94912225e-02,  1.44609317e-01, -1.13541201e-01,\n",
       "          -1.07075088e-01,  3.42293382e-02,  9.88678709e-02,\n",
       "          -1.50391042e-01,  9.53641012e-02, -1.06696282e-02,\n",
       "          -8.81695747e-02,  2.56545842e-02],\n",
       "         [-5.77516675e-01, -6.18089911e-01, -3.52066928e-01,\n",
       "          -4.75861800e-01,  4.96138097e-01,  4.59995451e-01,\n",
       "          -3.77813000e-01,  6.00455976e-01, -4.80238913e-01,\n",
       "          -6.69563201e-01, -4.74795816e-01,  4.62968927e-01,\n",
       "           4.11359972e-01,  4.43111042e-01,  3.59881510e-01,\n",
       "          -6.06320205e-01, -4.21140573e-01, -5.73674742e-01,\n",
       "           5.91024089e-01,  5.37596511e-01, -6.68298457e-01,\n",
       "           6.65245271e-01,  5.96190947e-01,  6.64344289e-01,\n",
       "           3.21412028e-01,  7.42421755e-01, -5.97499860e-01,\n",
       "           5.58377606e-01,  4.81321411e-01, -4.88104737e-01,\n",
       "           3.84848887e-01, -3.43747423e-01,  5.29766784e-01,\n",
       "          -5.43439470e-01, -5.25546634e-01, -3.09674530e-01,\n",
       "          -5.95669751e-01,  6.09231764e-01,  3.93370234e-01,\n",
       "           2.25453324e-01,  5.05395535e-01, -5.27451309e-01,\n",
       "           6.12555528e-01, -3.94444614e-01, -6.74554222e-01,\n",
       "           4.64775408e-01, -9.48248907e-02, -7.68106997e-01,\n",
       "          -5.72268437e-01,  4.92430475e-01, -6.50284640e-01,\n",
       "           5.57865407e-01,  4.93645664e-01,  5.20631009e-01,\n",
       "          -7.10935772e-01, -5.24166934e-01, -5.63016719e-01,\n",
       "           6.14007475e-01, -3.18349779e-01,  5.54783535e-01,\n",
       "           4.85022681e-01,  5.37467909e-01,  5.59441815e-01,\n",
       "          -5.04301906e-01, -5.37377762e-01,  5.47708543e-01,\n",
       "          -4.62781010e-01, -7.17099970e-01,  6.29946812e-01,\n",
       "          -4.78963461e-01, -5.40704428e-01, -3.66731808e-01,\n",
       "          -3.80452086e-01, -4.41724013e-01,  8.18987530e-01,\n",
       "          -4.64774673e-01, -5.79889722e-01,  6.69544320e-01,\n",
       "           2.45410488e-01,  4.52681415e-01, -5.00647311e-01,\n",
       "           3.20979641e-01,  4.61106458e-01, -4.27636043e-01,\n",
       "          -4.40790442e-01, -6.64471064e-01, -5.39096584e-01,\n",
       "          -4.64493383e-01, -4.75951124e-01,  5.14621023e-01,\n",
       "          -4.94421869e-01, -3.61153487e-01,  5.74876242e-01,\n",
       "           5.66119378e-01, -5.97063035e-01,  5.50503647e-01,\n",
       "           5.32057408e-01,  4.12173358e-01,  6.24858655e-01,\n",
       "           5.78696335e-01,  5.53475545e-01,  5.13225235e-01,\n",
       "           5.54929351e-01,  2.85364488e-01,  4.48318645e-01,\n",
       "          -4.59243853e-01, -5.38397295e-01, -5.90250714e-01,\n",
       "           4.54159756e-01,  4.05212476e-01,  7.05584590e-01,\n",
       "           5.25710743e-01,  3.92139717e-01, -5.64833457e-01,\n",
       "          -3.24598260e-01,  4.57404105e-01, -4.45824263e-01,\n",
       "          -5.76573958e-01, -5.55097708e-01, -7.43564099e-01,\n",
       "           4.16153088e-01,  5.14029055e-01,  5.90597933e-01,\n",
       "           3.45760307e-01, -1.09261320e-01, -5.75365886e-01,\n",
       "          -4.94889133e-01,  5.72729304e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01],\n",
       "         [ 4.71694444e-01,  3.19764686e-01,  5.44816471e-01,\n",
       "           6.13107223e-01,  5.67655652e-01, -4.73417875e-01,\n",
       "           7.41432835e-01, -4.10865369e-01,  4.66350568e-01,\n",
       "           5.42884630e-01,  4.28369224e-01,  5.45218986e-01,\n",
       "          -4.67836014e-01,  5.30379657e-01,  4.23243575e-01,\n",
       "          -2.63428372e-01,  4.89218153e-01, -5.52768254e-01,\n",
       "           4.43898282e-01,  4.47254614e-01, -4.77414675e-01,\n",
       "          -5.79248468e-01, -5.35537324e-01,  5.36308804e-01,\n",
       "           4.29858006e-01, -4.29957515e-01, -4.78773878e-01,\n",
       "           4.10313213e-01, -5.10452658e-01, -4.99651335e-01,\n",
       "           4.69268928e-01,  5.26401812e-01,  3.92788517e-01,\n",
       "          -4.69772932e-01,  5.47807893e-01, -6.13491949e-01,\n",
       "           5.46517200e-01,  5.45292195e-01, -6.45790812e-01,\n",
       "           4.34300279e-01,  5.09236181e-01, -3.87527918e-01,\n",
       "          -5.32055123e-01, -4.11784236e-01,  5.10244457e-01,\n",
       "           4.11306773e-01,  5.37123124e-01,  5.06752082e-01,\n",
       "          -5.06614187e-01,  5.56395845e-01, -5.46491171e-01,\n",
       "           3.40444284e-01,  4.13056324e-01,  5.60318265e-01,\n",
       "          -5.30808755e-01, -5.78415922e-01, -5.27163219e-01,\n",
       "          -6.51290794e-01,  7.91848133e-01,  5.03531796e-01,\n",
       "           4.97909308e-01, -6.37897894e-01,  6.30755849e-01,\n",
       "          -1.84361682e-01,  4.14885519e-01,  5.18906396e-01,\n",
       "          -4.04610406e-01,  4.71218326e-01,  6.45883510e-01,\n",
       "          -6.31970612e-01, -5.73774084e-01,  5.89063242e-01,\n",
       "          -5.57959069e-01,  5.80181429e-01,  4.82674786e-01,\n",
       "          -3.52797834e-01,  5.65930896e-01, -3.49473874e-01,\n",
       "          -2.36327547e-02,  3.64674765e-01, -5.43330407e-01,\n",
       "           4.74873155e-01,  4.54098465e-01, -4.68340507e-01,\n",
       "           5.95785799e-01,  4.51114589e-01,  4.47558449e-01,\n",
       "           5.27910037e-01,  5.77100835e-01, -4.17445844e-01,\n",
       "           5.63970317e-01,  6.46804339e-01,  5.58569781e-01,\n",
       "          -5.16361534e-01,  6.35673185e-01,  2.28088160e-01,\n",
       "          -3.68949672e-01,  5.57374673e-01,  6.51767878e-01,\n",
       "          -4.20929354e-01, -5.36897577e-01, -6.04019444e-01,\n",
       "          -5.80638498e-01,  2.01029854e-01, -6.32111781e-01,\n",
       "           6.30472135e-01, -4.76525547e-01, -5.86207485e-01,\n",
       "           4.55425970e-01,  4.23377543e-01, -5.81796765e-01,\n",
       "           5.60360358e-01, -5.89767200e-01,  4.63174580e-01,\n",
       "          -4.44858653e-01, -4.95635470e-01, -2.78127258e-01,\n",
       "           5.13800491e-01,  5.63854463e-01,  4.74592299e-01,\n",
       "          -5.03997386e-01,  5.02016542e-01, -5.03973387e-01,\n",
       "           3.27363062e-01,  5.96663460e-01, -6.77362753e-01,\n",
       "           5.42481347e-01,  5.17745441e-01]]]),\n",
       " 'choices_split_words_ids_tensor': array([[    1,  2460,  6595,     2,     0,     0,     0,     0,     0],\n",
       "        [    1,  1599,     2,     0,     0,     0,     0,     0,     0],\n",
       "        [    1,  9277,     2,     0,     0,     0,     0,     0,     0],\n",
       "        [    1, 12856,     2,     0,     0,     0,     0,     0,     0]]),\n",
       " 'choices_outputs': array([[[-6.87924809e-10, -8.05151442e-18,  6.53057500e-21,\n",
       "           2.16833048e-10, -4.89407796e-13,  1.04547698e-17,\n",
       "           0.00000000e+00, -5.00940091e-19, -8.80148817e-19,\n",
       "           1.02813174e-18, -4.14987662e-19,  1.08011197e-20,\n",
       "          -1.09411841e-12,  1.09441140e-16, -1.50255326e-17,\n",
       "           9.54536452e-21,  4.69587652e-19,  8.01677748e-01,\n",
       "          -1.58838115e-11,  0.00000000e+00, -3.87795033e-15,\n",
       "           6.66277268e-13, -1.53634724e-20, -1.65694859e-16,\n",
       "           8.61096227e-11,  5.25844526e-10,  4.29367994e-11,\n",
       "          -8.01678069e-01, -7.82817781e-19,  4.22521309e-13,\n",
       "           9.69449410e-20,  2.94359019e-07,  1.74702366e-12,\n",
       "           1.36678678e-20,  1.51961009e-13,  2.32332090e-18,\n",
       "          -8.01678069e-01, -8.81807200e-19,  6.36070881e-13,\n",
       "           2.25554757e-11, -9.48836858e-20, -2.18731656e-18,\n",
       "          -1.18639128e-05,  3.46050919e-16, -2.30641072e-18,\n",
       "           9.69292503e-25, -6.87938836e-20, -8.01512656e-01,\n",
       "          -5.21421624e-11, -0.00000000e+00,  1.28848410e-20,\n",
       "           6.54710467e-20,  8.03643760e-13,  4.23851083e-22,\n",
       "          -3.08363242e-20, -9.27282336e-23, -4.65018360e-18,\n",
       "           4.98256026e-18,  7.02831632e-18,  1.21656828e-20,\n",
       "          -1.58866248e-19,  1.88166782e-23,  4.31342462e-10,\n",
       "           9.76412104e-19],\n",
       "         [-5.50968684e-14, -3.83450344e-25,  2.02258149e-27,\n",
       "           8.02075775e-15, -1.64624940e-18,  4.62773203e-25,\n",
       "           5.78070698e-18, -1.51349430e-26, -2.39341261e-26,\n",
       "           3.04185862e-27, -1.33828993e-26,  2.39689831e-26,\n",
       "          -2.58540506e-17,  4.27271660e-23, -1.33580735e-24,\n",
       "           0.00000000e+00,  1.90034019e-26,  1.01476588e+00,\n",
       "          -6.72570870e-16,  1.01476589e+00, -1.54280182e-18,\n",
       "           2.94541028e-18, -1.57732691e-26, -8.16264355e-23,\n",
       "           7.24865629e-15,  3.12701097e-14,  5.83340434e-16,\n",
       "          -1.01476589e+00, -2.39642199e-26,  1.29051281e-18,\n",
       "           2.26523272e-26,  1.76086012e-10,  0.00000000e+00,\n",
       "           8.82540865e-27,  1.15886897e-18,  8.71787788e-26,\n",
       "          -1.01476589e+00, -1.72605232e-26,  3.10374864e-18,\n",
       "           2.18427209e-16, -2.40914793e-27, -8.36541279e-26,\n",
       "          -3.77450781e-07,  8.20849921e-23, -3.96220565e-24,\n",
       "           9.83767260e-33, -0.00000000e+00, -1.01473136e+00,\n",
       "          -1.12845694e-15, -4.09665653e-26,  6.31350489e-29,\n",
       "           6.14071244e-27,  3.69354820e-18,  5.03861531e-30,\n",
       "          -3.53886797e-28, -5.79138517e-32, -8.83907890e-25,\n",
       "           7.52890454e-25,  3.47745505e-25,  3.68278812e-24,\n",
       "          -1.07010075e-26,  6.91216894e-32,  2.97201573e-14,\n",
       "           3.49645885e-26],\n",
       "         [-7.68636051e-15, -0.00000000e+00,  1.26523145e-28,\n",
       "           1.18016785e-15, -1.02360882e-19,  2.34728679e-25,\n",
       "           3.26027156e-19, -7.12392152e-27, -1.46456255e-26,\n",
       "           1.01915051e-28, -7.36170469e-27,  2.40776489e-27,\n",
       "          -4.14931167e-18,  4.01493279e-23, -9.00155118e-25,\n",
       "           3.16157107e-31,  9.83267869e-27,  1.04742607e+00,\n",
       "          -1.00449025e-16,  1.04742607e+00, -6.90325738e-19,\n",
       "           0.00000000e+00, -1.57274529e-27, -5.77093923e-23,\n",
       "           1.22142422e-15,  4.58277260e-15,  5.34906631e-17,\n",
       "          -1.04742607e+00, -1.59535068e-26,  8.72680994e-20,\n",
       "           4.81227945e-27,  1.58519063e-11,  1.28370571e-17,\n",
       "           6.70760016e-28,  1.11934955e-18,  5.33821826e-26,\n",
       "          -1.04742607e+00, -9.14127122e-27,  2.38666432e-19,\n",
       "           1.70116566e-17, -4.26111694e-28, -5.16465778e-26,\n",
       "          -1.59334842e-07,  5.21633726e-23, -7.78768912e-25,\n",
       "           2.46390923e-34, -1.26702493e-27, -1.04735135e+00,\n",
       "          -9.76750566e-17, -2.29731369e-26,  0.00000000e+00,\n",
       "           1.97141906e-28,  2.91093934e-19,  1.40377453e-31,\n",
       "          -7.10860940e-30, -1.26317359e-33, -3.31566492e-25,\n",
       "           3.96304689e-25,  2.07082141e-25,  6.78871703e-25,\n",
       "          -3.17100586e-27,  1.44024349e-33,  4.38798793e-15,\n",
       "           1.96728878e-26],\n",
       "         [-1.29680628e-14, -0.00000000e+00,  3.51115973e-28,\n",
       "           1.70630964e-15, -2.38682894e-19,  5.24808141e-25,\n",
       "           6.31547251e-19, -2.13300200e-26, -3.09041033e-26,\n",
       "           2.36145307e-28, -2.13133277e-26,  3.96785886e-27,\n",
       "          -7.10130358e-18,  9.06840155e-23, -2.96283617e-24,\n",
       "           9.24435356e-31,  2.98586050e-26,  1.05192559e+00,\n",
       "          -1.89343271e-16,  1.05192559e+00, -7.53511844e-19,\n",
       "           3.73266030e-19, -3.59496843e-27, -0.00000000e+00,\n",
       "           1.54580691e-15,  7.86513639e-15,  9.79996397e-17,\n",
       "          -1.05192559e+00, -4.75685304e-26,  1.83766632e-19,\n",
       "           0.00000000e+00,  4.25861075e-11,  2.34894631e-17,\n",
       "           2.12271256e-27,  4.84441375e-19,  1.34347982e-25,\n",
       "          -1.05192559e+00, -2.91057414e-26,  4.93029062e-19,\n",
       "           3.24180437e-17, -1.17230132e-27, -0.00000000e+00,\n",
       "          -2.70101717e-07,  1.36423725e-22, -1.40890067e-24,\n",
       "           4.65831389e-34, -3.73229293e-27, -1.05187493e+00,\n",
       "          -0.00000000e+00, -6.20994924e-26,  3.80183509e-30,\n",
       "           3.46622341e-28,  4.99162337e-19,  0.00000000e+00,\n",
       "          -1.52465060e-29, -2.83200437e-33, -1.06744287e-24,\n",
       "           1.16164766e-24,  8.39736797e-25,  8.04942167e-25,\n",
       "          -8.41128569e-27,  0.00000000e+00,  6.64041468e-15,\n",
       "           5.50516858e-26],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00]],\n",
       " \n",
       "        [[-7.25408602e-10, -1.61376589e-18,  6.72312469e-21,\n",
       "           0.00000000e+00, -5.16988060e-13,  3.08682364e-18,\n",
       "           1.83713692e-12, -2.63611579e-19, -3.99282841e-19,\n",
       "           0.00000000e+00, -3.67417368e-19,  9.23754740e-21,\n",
       "          -1.15951271e-12,  7.17291382e-17, -6.65344594e-18,\n",
       "           0.00000000e+00,  2.53806475e-19,  8.01677724e-01,\n",
       "          -1.83663965e-11,  8.01678069e-01, -3.50155920e-15,\n",
       "           7.37584959e-13, -1.75225272e-20, -7.97812500e-17,\n",
       "           6.94700685e-11,  5.49902476e-10,  4.82454679e-11,\n",
       "          -8.01678069e-01, -3.98257078e-19,  3.86592895e-13,\n",
       "           1.04064298e-19,  3.97705194e-07,  2.51393981e-12,\n",
       "           1.27403289e-20,  1.28834906e-13,  1.04646554e-18,\n",
       "          -8.01678069e-01, -3.51974173e-19,  7.84967809e-13,\n",
       "           2.17407007e-11, -0.00000000e+00, -7.76526909e-19,\n",
       "          -2.36362610e-05,  9.67589214e-17, -2.46158048e-18,\n",
       "           1.34451709e-24, -7.35096527e-20, -8.01524695e-01,\n",
       "          -5.64618156e-11, -7.01006187e-19,  9.16868942e-21,\n",
       "           4.67772760e-20,  9.29029527e-13,  3.87379961e-22,\n",
       "          -2.32365726e-20, -1.05608647e-22, -3.98366087e-18,\n",
       "           3.50963821e-18,  2.95208697e-18,  1.21561222e-20,\n",
       "          -1.80514977e-19,  1.73725124e-23,  4.48292112e-10,\n",
       "           4.36638527e-19],\n",
       "         [-2.71557945e-13, -8.16634144e-23,  5.89133981e-26,\n",
       "           4.09283025e-14, -1.25517001e-17,  8.84939596e-23,\n",
       "           3.58391707e-17, -4.29765332e-24, -7.69407095e-24,\n",
       "           7.34380819e-26, -3.50037075e-24,  6.87609877e-25,\n",
       "          -1.67746917e-16,  6.42353349e-21, -0.00000000e+00,\n",
       "           3.99180528e-28,  0.00000000e+00,  1.01476588e+00,\n",
       "          -3.89712450e-15,  1.01476589e+00, -5.84357679e-18,\n",
       "           2.60882187e-17, -3.98433813e-25, -1.37084290e-20,\n",
       "           0.00000000e+00,  1.12349197e-13,  3.37785969e-15,\n",
       "          -1.01476589e+00, -5.12315860e-24,  9.61946845e-18,\n",
       "           1.67393144e-24,  4.70022790e-10,  5.77360157e-16,\n",
       "           1.96026507e-25,  1.65387092e-16,  1.86352612e-23,\n",
       "          -1.01476589e+00, -3.86805932e-24,  1.60253971e-17,\n",
       "           1.00073921e-15, -2.50798816e-25, -2.04797749e-23,\n",
       "          -7.65021545e-07,  1.75295775e-20, -6.88894746e-23,\n",
       "           7.16050525e-31, -6.15824419e-25, -1.01469592e+00,\n",
       "          -5.87502845e-15, -1.06226342e-23,  1.45979811e-27,\n",
       "           8.46264921e-26,  3.69251845e-17,  1.24633955e-28,\n",
       "          -5.45767514e-27, -0.00000000e+00, -9.09332663e-23,\n",
       "           8.62991196e-23,  1.06170318e-22,  9.54263965e-24,\n",
       "          -1.31135091e-24,  2.09809352e-30,  1.10253502e-13,\n",
       "           7.88207958e-24],\n",
       "         [-1.55717608e-14, -2.83386148e-25,  4.67984953e-28,\n",
       "           2.26911666e-15, -2.94163017e-19,  3.34650243e-25,\n",
       "           8.04355846e-19, -2.62356018e-26, -5.04433526e-26,\n",
       "           3.00316469e-28, -2.71055903e-26,  1.04726987e-26,\n",
       "          -9.94142568e-18,  1.17740461e-22, -2.75296881e-24,\n",
       "           0.00000000e+00,  4.15562057e-26,  1.04742606e+00,\n",
       "          -2.69981633e-16,  1.04742607e+00, -2.75762619e-18,\n",
       "           5.43119798e-19, -7.42711807e-27, -5.19324416e-23,\n",
       "           2.19700550e-15,  1.00145630e-14,  1.16454714e-16,\n",
       "          -1.04742607e+00, -6.11071184e-26,  2.30358515e-19,\n",
       "           2.69718015e-26,  3.14244443e-11,  3.62182667e-17,\n",
       "           2.91622409e-27,  4.89685966e-19,  1.90500593e-25,\n",
       "          -1.04742607e+00, -3.65229430e-26,  5.89627944e-19,\n",
       "           3.60416113e-17, -2.20506745e-27, -2.15962314e-25,\n",
       "          -2.99132729e-07,  0.00000000e+00, -2.17507243e-24,\n",
       "           1.28702882e-33, -5.32305452e-27, -1.04734214e+00,\n",
       "          -2.13012308e-16, -8.59681764e-26,  8.61744869e-30,\n",
       "           1.01500282e-27,  7.26669694e-19,  6.56025577e-31,\n",
       "          -3.10919121e-29, -5.04582052e-33, -1.01137447e-24,\n",
       "           9.75152256e-25,  9.91301391e-25,  1.44051555e-24,\n",
       "          -1.38555665e-26,  8.01563893e-33,  7.07961331e-15,\n",
       "           7.00003003e-26],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00]],\n",
       " \n",
       "        [[-3.19713307e-10, -1.42305476e-18,  1.12028454e-21,\n",
       "           0.00000000e+00, -1.82490761e-13,  2.80640498e-18,\n",
       "           0.00000000e+00, -9.09759764e-20, -1.59157182e-19,\n",
       "           2.11812183e-19, -7.44564428e-20,  0.00000000e+00,\n",
       "          -3.06999155e-13,  2.50249170e-17, -3.00133884e-18,\n",
       "           1.53306879e-21,  9.55107417e-20,  8.01677869e-01,\n",
       "          -5.46129074e-12,  8.01678069e-01, -9.73303979e-16,\n",
       "           2.36967172e-13, -2.66052709e-21, -6.79549322e-17,\n",
       "           3.54347741e-11,  2.21578397e-10,  1.98729019e-11,\n",
       "          -0.00000000e+00, -1.59938885e-19,  1.53790264e-13,\n",
       "           1.99809900e-20,  1.40095985e-07,  4.81921104e-13,\n",
       "           2.09302717e-21,  1.55203639e-13,  3.81151629e-19,\n",
       "          -8.01678069e-01, -1.49340751e-19,  2.23425952e-13,\n",
       "           8.28848183e-12, -1.47605910e-20, -3.65911553e-19,\n",
       "          -7.80029296e-06,  1.02619907e-16, -4.03156029e-19,\n",
       "           1.20939515e-25, -1.17402300e-20, -8.01526526e-01,\n",
       "          -2.14887148e-11, -3.25772020e-19,  2.02423797e-21,\n",
       "           1.10726251e-20,  2.81061238e-13,  7.30286810e-23,\n",
       "          -4.94157782e-21, -1.68066958e-23, -9.83025218e-19,\n",
       "           1.05471801e-18,  1.36237010e-18,  2.24556866e-21,\n",
       "          -2.98395566e-20,  2.96507652e-24,  1.99226101e-10,\n",
       "           1.72395349e-19],\n",
       "         [-9.62295865e-14, -6.69787281e-24,  6.35368197e-27,\n",
       "           1.31757188e-14, -2.66171220e-18,  1.08731407e-23,\n",
       "           9.21216789e-18, -5.06034130e-25, -7.31747809e-25,\n",
       "           9.65469023e-27, -2.90962390e-25,  7.10305862e-26,\n",
       "          -3.39422476e-17,  1.21307800e-21, -3.75423172e-23,\n",
       "           3.73124622e-29,  4.99331760e-25,  1.01476587e+00,\n",
       "          -8.68058970e-16,  1.01476589e+00, -3.74932456e-18,\n",
       "           5.15155640e-18, -5.26074052e-26, -0.00000000e+00,\n",
       "           1.08172057e-14,  4.93039660e-14,  9.12978245e-16,\n",
       "          -1.01476589e+00, -6.58399939e-25,  2.41195899e-18,\n",
       "           0.00000000e+00,  1.58975767e-10,  9.53952285e-17,\n",
       "           0.00000000e+00,  2.99210818e-17,  3.00951330e-24,\n",
       "          -1.01476589e+00, -5.91294482e-25,  5.28038414e-18,\n",
       "           0.00000000e+00, -2.57530278e-26, -2.34295793e-24,\n",
       "          -2.53089957e-07,  2.39311804e-21, -1.83957233e-23,\n",
       "           4.11333346e-32, -0.00000000e+00, -1.01468012e+00,\n",
       "          -1.50465170e-15, -1.16080898e-24,  0.00000000e+00,\n",
       "           8.65966587e-27,  7.85730456e-18,  1.13605837e-29,\n",
       "          -6.47279100e-28, -3.90258974e-31, -1.30524026e-23,\n",
       "           1.36382428e-23,  1.13697282e-23,  2.00387418e-24,\n",
       "          -0.00000000e+00,  1.90484391e-31,  4.67423574e-14,\n",
       "           8.95207888e-25],\n",
       "         [-4.48266691e-14, -6.72784921e-24,  3.14254234e-27,\n",
       "           8.01096927e-15, -1.08573376e-18,  9.01957249e-24,\n",
       "           4.87299336e-18, -4.73201559e-25, -4.64276623e-25,\n",
       "           4.24869071e-27, -2.43139862e-25,  8.16238156e-27,\n",
       "          -1.98014432e-17,  6.81034029e-22, -3.60655091e-23,\n",
       "           1.07539770e-29,  3.93556586e-25,  1.04742606e+00,\n",
       "          -6.69315270e-16,  1.04742607e+00, -5.50138506e-19,\n",
       "           2.53192064e-18, -1.75076437e-26, -1.66935062e-21,\n",
       "           6.15436621e-15,  2.77885073e-14,  3.70741267e-16,\n",
       "          -1.04742607e+00, -5.85118122e-25,  1.08655299e-18,\n",
       "           1.53371157e-25,  1.22049402e-10,  4.84442980e-17,\n",
       "           1.79713970e-26,  4.70371061e-17,  2.03684349e-24,\n",
       "          -1.04742607e+00, -0.00000000e+00,  2.34859307e-18,\n",
       "           1.63584874e-16, -2.15232721e-26, -1.62632479e-24,\n",
       "          -3.70700922e-07,  2.05770905e-21, -6.67576246e-24,\n",
       "           9.64710523e-33, -3.48072059e-26, -1.04737211e+00,\n",
       "          -7.67205797e-16, -8.07844136e-25,  0.00000000e+00,\n",
       "           1.97779200e-27,  3.25429644e-18,  2.07928283e-30,\n",
       "          -1.55818432e-28, -1.38659272e-31, -0.00000000e+00,\n",
       "           1.17470913e-23,  0.00000000e+00,  7.42958652e-25,\n",
       "          -1.04353158e-25,  3.63926039e-32,  3.19727537e-14,\n",
       "           4.95760770e-25],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00]],\n",
       " \n",
       "        [[-8.95578658e-10, -0.00000000e+00,  1.26207394e-20,\n",
       "           3.13935023e-10, -8.19376064e-13,  2.67579376e-17,\n",
       "           2.36854954e-12, -9.47709622e-19, -0.00000000e+00,\n",
       "           1.77088048e-18, -8.49877120e-19,  0.00000000e+00,\n",
       "          -8.95929758e-13,  2.44129142e-16, -3.47092541e-17,\n",
       "           0.00000000e+00,  9.06986639e-19,  8.01677410e-01,\n",
       "          -1.58041118e-11,  0.00000000e+00, -9.18123854e-15,\n",
       "           1.03516531e-12, -4.75158568e-20, -7.24083288e-16,\n",
       "           1.40114725e-10,  7.56456362e-10,  6.12150105e-11,\n",
       "          -8.01678069e-01, -2.19693879e-18,  6.71314917e-13,\n",
       "           3.37539182e-19,  2.08929259e-07,  1.55120805e-12,\n",
       "           3.16223573e-20,  1.96606315e-12,  5.32849825e-18,\n",
       "          -8.01678069e-01, -1.79850146e-18,  8.52031279e-13,\n",
       "           2.37967033e-11, -1.95074918e-19, -6.00785081e-18,\n",
       "          -1.20171692e-05,  1.32473885e-15, -4.90900056e-18,\n",
       "           2.14992074e-24, -1.61917623e-19, -8.01266265e-01,\n",
       "          -6.21531357e-11, -4.24138973e-18,  3.74480922e-20,\n",
       "           1.36582576e-19,  1.08583112e-12,  1.20189022e-21,\n",
       "          -6.84453825e-20, -2.45399502e-22, -9.51375156e-18,\n",
       "           1.14044422e-17,  1.59529579e-17,  2.86825302e-20,\n",
       "          -3.47254945e-19,  5.58449343e-23,  5.84588268e-10,\n",
       "           1.98627959e-18],\n",
       "         [-9.94430605e-14, -1.43328327e-24,  3.85357144e-27,\n",
       "           1.07848773e-14, -2.75780652e-18,  1.53498982e-24,\n",
       "           9.52651336e-18, -4.99583840e-26, -1.04291232e-25,\n",
       "           6.75771953e-27, -0.00000000e+00,  5.85520543e-26,\n",
       "          -2.25433156e-17,  2.22010756e-22, -4.85152163e-24,\n",
       "           3.36133764e-29,  7.91809846e-26,  1.01476588e+00,\n",
       "          -8.49977349e-16,  1.01476589e+00, -6.66071961e-18,\n",
       "           5.01414147e-18, -4.35541695e-26, -2.80340228e-22,\n",
       "           9.53737222e-15,  4.06982317e-14,  7.79667547e-16,\n",
       "          -1.01476589e+00, -1.06163479e-25,  0.00000000e+00,\n",
       "           6.57489635e-26,  1.95094974e-10,  8.73649047e-17,\n",
       "           1.46828782e-26,  4.29876204e-18,  2.57212201e-25,\n",
       "          -1.01476589e+00, -5.84873521e-26,  4.96664188e-18,\n",
       "           2.93897327e-16, -4.48036175e-27, -3.24197950e-25,\n",
       "          -3.62257265e-07,  3.88819570e-22, -8.02351752e-24,\n",
       "           2.25489617e-32, -2.83162967e-26, -1.01471111e+00,\n",
       "          -1.42042929e-15, -1.57769078e-25,  1.47554172e-28,\n",
       "           1.33805172e-26,  4.31056803e-18,  1.62986729e-29,\n",
       "          -8.29700497e-28, -1.63153567e-31, -1.75254284e-24,\n",
       "           2.55107791e-24,  1.49598785e-24,  9.09294592e-24,\n",
       "          -2.64246619e-26,  0.00000000e+00,  4.18264838e-14,\n",
       "           0.00000000e+00],\n",
       "         [-5.70074084e-15, -7.13434392e-26,  4.85104406e-29,\n",
       "           7.46619979e-16, -0.00000000e+00,  8.90450254e-26,\n",
       "           2.21555801e-19, -2.99389449e-27, -0.00000000e+00,\n",
       "           4.33400783e-29, -2.84290468e-27,  1.05337317e-27,\n",
       "          -1.57143420e-18,  1.99466423e-23, -3.99555358e-25,\n",
       "           1.30465376e-31,  4.18804578e-27,  1.04742607e+00,\n",
       "          -5.25243955e-17,  1.04742607e+00, -3.52322397e-19,\n",
       "           1.33076080e-19, -6.21046625e-28, -2.21135393e-23,\n",
       "           7.49512295e-16,  3.19998810e-15,  3.77827720e-17,\n",
       "          -1.04742607e+00, -6.24343189e-27,  5.61047504e-20,\n",
       "           2.05928903e-27,  1.58901084e-11,  5.93596125e-18,\n",
       "           2.79797289e-28,  4.69155138e-19,  2.01893264e-26,\n",
       "          -1.04742607e+00, -3.51729946e-27,  1.58058813e-19,\n",
       "           1.19583851e-17, -1.82168399e-28, -2.04870233e-26,\n",
       "          -1.41423373e-07,  2.27341089e-23, -2.91367044e-25,\n",
       "           9.23780081e-35, -5.70033418e-28, -1.04737534e+00,\n",
       "          -6.60492218e-17, -8.66079829e-27,  6.30529070e-31,\n",
       "           8.00727732e-29,  1.76786180e-19,  4.74151072e-32,\n",
       "          -3.21641499e-30, -4.29822338e-34, -1.28422888e-25,\n",
       "           0.00000000e+00,  1.00606072e-25,  5.83665553e-25,\n",
       "          -1.20387547e-27,  5.21598020e-34,  2.85665430e-15,\n",
       "           7.56799676e-27],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00],\n",
       "         [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "           0.00000000e+00]]])}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
